@redstone-md/mapr 0.0.1-alpha → 0.0.3-alpha

package/lib/ai-json.ts

@@ -0,0 +1,134 @@
+import { generateText, Output } from "ai";
+import { z } from "zod";
+
+const jsonFencePattern = /^```(?:json)?\s*([\s\S]*?)\s*```$/i;
+
+function extractBalancedJsonSlice(source: string): string | null {
+  const startIndex = source.search(/[\[{]/);
+  if (startIndex < 0) {
+    return null;
+  }
+
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let index = startIndex; index < source.length; index += 1) {
+    const character = source[index];
+
+    if (!character) {
+      continue;
+    }
+
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === "\"") {
+        inString = false;
+      }
+
+      continue;
+    }
+
+    if (character === "\"") {
+      inString = true;
+      continue;
+    }
+
+    if (character === "{" || character === "[") {
+      depth += 1;
+      continue;
+    }
+
+    if (character === "}" || character === "]") {
+      depth -= 1;
+      if (depth === 0) {
+        return source.slice(startIndex, index + 1);
+      }
+    }
+  }
+
+  return null;
+}
+
+export function extractJsonFromText(source: string): unknown {
+  const trimmed = source.trim();
+  if (!trimmed) {
+    throw new Error("Model returned empty text instead of JSON.");
+  }
+
+  const fenced = trimmed.match(jsonFencePattern)?.[1]?.trim();
+  const directCandidate = fenced ?? trimmed;
+
+  try {
+    return JSON.parse(directCandidate) as unknown;
+  } catch {
+    const balancedSlice = extractBalancedJsonSlice(directCandidate);
+    if (!balancedSlice) {
+      throw new Error("No JSON object found in model output.");
+    }
+
+    return JSON.parse(balancedSlice) as unknown;
+  }
+}
+
+export function shouldFallbackToTextJson(error: unknown): boolean {
+  if (!(error instanceof Error)) {
+    return false;
+  }
+
+  const message = error.message.toLowerCase();
+  return (
+    message.includes("responseformat") ||
+    message.includes("structured output") ||
+    message.includes("structuredoutputs") ||
+    message.includes("response did not match schema") ||
+    message.includes("no object generated")
+  );
+}
+
+export async function generateObjectWithTextFallback<TOutput>(input: {
+  model: unknown;
+  system: string;
+  prompt: string;
+  schema: z.ZodType<TOutput>;
+  contract: string;
+  maxRetries?: number;
+  providerOptions?: Record<string, unknown>;
+}): Promise<TOutput> {
+  try {
+    const structuredResult = await generateText({
+      model: input.model as never,
+      system: input.system,
+      prompt: input.prompt,
+      output: Output.object({ schema: input.schema }),
+      maxRetries: input.maxRetries ?? 2,
+      ...(input.providerOptions !== undefined ? { providerOptions: input.providerOptions as never } : {}),
+    });
+
+    return input.schema.parse(structuredResult.output);
+  } catch (error) {
+    if (!shouldFallbackToTextJson(error)) {
+      throw error;
+    }
+  }
+
+  const textResult = await generateText({
+    model: input.model as never,
+    system: [
+      input.system,
+      "Return only one valid JSON object.",
+      "Do not wrap the JSON in markdown fences.",
+      "Do not add explanations before or after the JSON.",
+      input.contract,
+    ].join("\n"),
+    prompt: input.prompt,
+    output: Output.text(),
+    maxRetries: input.maxRetries ?? 2,
+    ...(input.providerOptions !== undefined ? { providerOptions: input.providerOptions as never } : {}),
+  });
+
+  return input.schema.parse(extractJsonFromText(textResult.output));
+}

package/lib/analysis-schema.ts

@@ -0,0 +1,135 @@
+import { z } from "zod";
+
+import { artifactTypeSchema } from "./artifacts";
+
+export const entryPointSchema = z.object({
+  symbol: z.string().min(1),
+  description: z.string().min(1),
+  evidence: z.string().min(1),
+});
+
+export const callGraphEdgeSchema = z.object({
+  caller: z.string().min(1),
+  callee: z.string().min(1),
+  rationale: z.string().min(1),
+});
+
+export const renamedSymbolSchema = z.object({
+  originalName: z.string().min(1),
+  suggestedName: z.string().min(1),
+  justification: z.string().min(1),
+});
+
+export const agentMemoSchema = z.object({
+  role: z.string().min(1),
+  summary: z.string().min(1),
+  observations: z.array(z.string().min(1)).default([]),
+  evidence: z.array(z.string().min(1)).default([]),
+  nextQuestions: z.array(z.string().min(1)).default([]),
+});
+
+export const chunkAnalysisSchema = z.object({
+  entryPoints: z.array(entryPointSchema).default([]),
+  initializationFlow: z.array(z.string().min(1)).default([]),
+  callGraph: z.array(callGraphEdgeSchema).default([]),
+  restoredNames: z.array(renamedSymbolSchema).default([]),
+  summary: z.string().min(1),
+  notableLibraries: z.array(z.string().min(1)).default([]),
+  investigationTips: z.array(z.string().min(1)).default([]),
+  risks: z.array(z.string().min(1)).default([]),
+});
+
+export const artifactSummarySchema = z.object({
+  url: z.string().url(),
+  type: artifactTypeSchema,
+  chunkCount: z.number().int().nonnegative(),
+  summary: z.string().min(1),
+});
+
+export const finalAnalysisSchema = z.object({
+  overview: z.string().min(1),
+  entryPoints: z.array(entryPointSchema).default([]),
+  initializationFlow: z.array(z.string().min(1)).default([]),
+  callGraph: z.array(callGraphEdgeSchema).default([]),
+  restoredNames: z.array(renamedSymbolSchema).default([]),
+  notableLibraries: z.array(z.string().min(1)).default([]),
+  investigationTips: z.array(z.string().min(1)).default([]),
+  risks: z.array(z.string().min(1)).default([]),
+  artifactSummaries: z.array(artifactSummarySchema),
+  analyzedChunkCount: z.number().int().nonnegative(),
+});
+
+export type BundleAnalysis = z.infer<typeof finalAnalysisSchema>;
+export type AgentMemo = z.infer<typeof agentMemoSchema>;
+export type ChunkAnalysis = z.infer<typeof chunkAnalysisSchema>;
+export type ArtifactSummary = z.infer<typeof artifactSummarySchema>;
+
+function deduplicate<T>(items: T[], keySelector: (item: T) => string): T[] {
+  const seen = new Set<string>();
+  const deduplicated: T[] = [];
+
+  for (const item of items) {
+    const key = keySelector(item);
+    if (seen.has(key)) {
+      continue;
+    }
+
+    seen.add(key);
+    deduplicated.push(item);
+  }
+
+  return deduplicated;
+}
+
+export function buildAnalysisSnapshot(input: {
+  overview: string;
+  artifactSummaries?: ArtifactSummary[];
+  chunkAnalyses?: ChunkAnalysis[];
+}): BundleAnalysis {
+  const artifactSummaries = input.artifactSummaries ?? [];
+  const chunkAnalyses = input.chunkAnalyses ?? [];
+
+  return finalAnalysisSchema.parse({
+    overview: input.overview,
+    entryPoints: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.entryPoints),
+      (entryPoint) => `${entryPoint.symbol}:${entryPoint.description}`,
+    ),
+    initializationFlow: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.initializationFlow),
+      (step) => step,
+    ),
+    callGraph: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.callGraph),
+      (edge) => `${edge.caller}->${edge.callee}`,
+    ),
+    restoredNames: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.restoredNames),
+      (entry) => `${entry.originalName}:${entry.suggestedName}`,
+    ),
+    notableLibraries: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.notableLibraries),
+      (library) => library,
+    ),
+    investigationTips: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.investigationTips),
+      (tip) => tip,
+    ),
+    risks: deduplicate(
+      chunkAnalyses.flatMap((analysis) => analysis.risks),
+      (risk) => risk,
+    ),
+    artifactSummaries,
+    analyzedChunkCount: chunkAnalyses.length,
+  });
+}
+
+export class PartialAnalysisError extends Error {
+  public readonly partialAnalysis: BundleAnalysis;
+
+  public constructor(message: string, partialAnalysis: BundleAnalysis) {
+    super(message);
+    this.name = "PartialAnalysisError";
+    this.partialAnalysis = partialAnalysis;
+  }
+}

package/lib/artifacts.ts

@@ -6,9 +6,15 @@ export const artifactTypeSchema = z.enum([
   "script",
   "service-worker",
   "worker",
-  "stylesheet",
-  "manifest",
-  "json",
+  "source-map",
+  "wasm",
+]);
+
+export const analyzableArtifactTypeSchema = z.enum([
+  "script",
+  "service-worker",
+  "worker",
+  "source-map",
   "wasm",
 ]);
 
@@ -30,6 +36,11 @@ export type ArtifactType = z.infer<typeof artifactTypeSchema>;
 export type DiscoveredArtifact = z.infer<typeof discoveredArtifactSchema>;
 export type ArtifactCandidate = z.infer<typeof artifactCandidateSchema>;
 
+const binaryOrVisualAssetPattern =
+  /\.(?:png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|mp4|webm|mov|avi|mp3|wav|ogg|flac|aac|m4a|pdf|zip|gz|tar|7z|rar|woff2?|ttf|otf|eot)(?:$|[?#])/i;
+const ignoredContentTypePattern =
+  /^(?:image\/|audio\/|video\/|font\/|application\/(?:font|octet-stream|pdf|zip|gzip|x-font|vnd\.ms-fontobject))/i;
+
 function makeCandidate(url: string, type: ArtifactType, discoveredFrom: string): ArtifactCandidate | null {
   const parsed = artifactCandidateSchema.safeParse({ url, type, discoveredFrom });
   return parsed.success ? parsed.data : null;
@@ -61,6 +72,10 @@ function resolveCandidateUrl(reference: string, baseUrl: string): string | null
 
   try {
     const absoluteUrl = new URL(reference, baseUrl).toString();
+    if (binaryOrVisualAssetPattern.test(new URL(absoluteUrl).pathname)) {
+      return null;
+    }
+
     const parsed = z.string().url().safeParse(absoluteUrl);
     return parsed.success ? parsed.data : null;
   } catch {
@@ -75,16 +90,8 @@ function inferAssetTypeFromUrl(url: string, fallback: ArtifactType = "script"):
     return "wasm";
   }
 
-  if (pathname.endsWith(".css")) {
-    return "stylesheet";
-  }
-
-  if (pathname.endsWith(".json")) {
-    return "json";
-  }
-
-  if (pathname.endsWith(".webmanifest") || pathname.endsWith("manifest.json")) {
-    return "manifest";
+  if (pathname.endsWith(".map")) {
+    return "source-map";
   }
 
   if (pathname.endsWith(".html") || pathname.endsWith(".htm")) {
@@ -94,14 +101,34 @@ function inferAssetTypeFromUrl(url: string, fallback: ArtifactType = "script"):
   return fallback;
 }
 
+function extractPageCandidate(reference: string, pageUrl: string, discoveredFrom: string): ArtifactCandidate | null {
+  const resolvedUrl = resolveCandidateUrl(reference, pageUrl);
+  if (!resolvedUrl) {
+    return null;
+  }
+
+  const pathname = new URL(resolvedUrl).pathname.toLowerCase();
+  const looksLikePage =
+    pathname === "" ||
+    pathname.endsWith("/") ||
+    pathname.endsWith(".html") ||
+    pathname.endsWith(".htm") ||
+    !/\.[a-z0-9]+$/i.test(pathname);
+
+  return looksLikePage ? makeCandidate(resolvedUrl, "html", discoveredFrom) : null;
+}
+
 function extractFromJavaScript(source: string, baseUrl: string, discoveredFrom: string): ArtifactCandidate[] {
   const candidates = new Map<string, ArtifactCandidate>();
   const regexDefinitions: Array<{ regex: RegExp; type: ArtifactType }> = [
     { regex: /(?:import|export)\s+(?:[^"'`]+?\s+from\s+)?["'`]([^"'`]+)["'`]/g, type: "script" },
     { regex: /import\(\s*["'`]([^"'`]+)["'`]\s*\)/g, type: "script" },
+    { regex: /importScripts\(\s*["'`]([^"'`]+)["'`]\s*\)/g, type: "script" },
     { regex: /navigator\.serviceWorker\.register\(\s*(?:new\s+URL\(\s*)?["'`]([^"'`]+)["'`]/g, type: "service-worker" },
     { regex: /new\s+(?:SharedWorker|Worker)\(\s*(?:new\s+URL\(\s*)?["'`]([^"'`]+)["'`]/g, type: "worker" },
-    { regex: /["'`]([^"'`]+\.wasm(?:\?[^"'`]*)?)["'`]/g, type: "wasm" },
+    { regex: /new\s+URL\(\s*["'`]([^"'`]+)["'`]\s*,\s*import\.meta\.url\s*\)/g, type: "script" },
+    { regex: /["'`]([^"'`]+\.(?:m?js|cjs|wasm|map)(?:\?[^"'`]*)?)["'`]/g, type: "script" },
+    { regex: /[@#]\s*sourceMappingURL=([^\s]+)/g, type: "source-map" },
   ];
 
   for (const definition of regexDefinitions) {
@@ -124,31 +151,12 @@ function extractFromJavaScript(source: string, baseUrl: string, discoveredFrom:
   return [...candidates.values()];
 }
 
-function extractFromCss(source: string, baseUrl: string, discoveredFrom: string): ArtifactCandidate[] {
-  const candidates = new Map<string, ArtifactCandidate>();
-  const regexDefinitions = [
-    /@import\s+(?:url\()?["']?([^"'()]+)["']?\)?/g,
-    /url\(\s*["']?([^"'()]+)["']?\s*\)/g,
-  ];
-
-  for (const regex of regexDefinitions) {
-    let match: RegExpExecArray | null;
-    while ((match = regex.exec(source)) !== null) {
-      const resolvedUrl = resolveCandidateUrl(match[1] ?? "", baseUrl);
-      if (!resolvedUrl) {
-        continue;
-      }
-
-      addCandidate(
-        candidates,
-        makeCandidate(resolvedUrl, inferAssetTypeFromUrl(resolvedUrl, "stylesheet"), discoveredFrom),
-        false,
-        new URL(baseUrl).origin,
-      );
-    }
-  }
+export function isAnalyzableArtifactType(type: ArtifactType): type is z.infer<typeof analyzableArtifactTypeSchema> {
+  return analyzableArtifactTypeSchema.safeParse(type).success;
+}
 
-  return [...candidates.values()];
+export function isIgnoredContentType(contentType: string): boolean {
+  return ignoredContentTypePattern.test(contentType.trim().toLowerCase());
 }
 
 export function extractArtifactCandidates(html: string, pageUrl: string): ArtifactCandidate[] {
@@ -170,40 +178,15 @@ export function extractArtifactCandidates(html: string, pageUrl: string): Artifa
     const rel = ($(element).attr("rel") ?? "").toLowerCase();
     const asValue = ($(element).attr("as") ?? "").toLowerCase();
 
-    if (rel.includes("manifest")) {
-      addCandidate(candidates, makeCandidate(href, "manifest", "html:manifest"), false, origin);
-      return;
-    }
-
-    if (rel.includes("stylesheet")) {
-      addCandidate(candidates, makeCandidate(href, "stylesheet", "html:stylesheet"), false, origin);
-      return;
-    }
-
     if (rel.includes("modulepreload") || (rel.includes("preload") && asValue === "script")) {
-      addCandidate(candidates, makeCandidate(href, "script", "html:preload"), false, origin);
+      addCandidate(candidates, makeCandidate(href, inferAssetTypeFromUrl(href, "script"), "html:preload"), false, origin);
     }
   });
 
-  $("a[href]").each((_, element) => {
-    const href = resolveCandidateUrl($(element).attr("href")?.trim() ?? "", pageUrl);
-    if (!href) {
-      return;
-    }
-
-    const url = new URL(href);
-    const isSameOrigin = url.origin === origin;
-    const pathname = url.pathname.toLowerCase();
-    const looksLikePage =
-      pathname === "" ||
-      pathname.endsWith("/") ||
-      pathname.endsWith(".html") ||
-      pathname.endsWith(".htm") ||
-      !/\.[a-z0-9]+$/i.test(pathname);
-
-    if (isSameOrigin && looksLikePage) {
-      addCandidate(candidates, makeCandidate(href, "html", "html:anchor"), true, origin);
-    }
+  $("a[href], iframe[src], form[action]").each((_, element) => {
+    const attributeName = element.tagName === "iframe" ? "src" : element.tagName === "form" ? "action" : "href";
+    const pageCandidate = extractPageCandidate($(element).attr(attributeName)?.trim() ?? "", pageUrl, `html:${element.tagName}`);
+    addCandidate(candidates, pageCandidate, true, origin);
   });
 
   $("script:not([src])").each((_, element) => {
@@ -221,13 +204,14 @@ export function extractNestedCandidates(artifact: DiscoveredArtifact): ArtifactC
     return extractArtifactCandidates(artifact.content, artifact.url);
   }
 
-  if (artifact.type === "script" || artifact.type === "service-worker" || artifact.type === "worker") {
+  if (
+    artifact.type === "script" ||
+    artifact.type === "service-worker" ||
+    artifact.type === "worker" ||
+    artifact.type === "source-map"
+  ) {
     return extractFromJavaScript(artifact.content, artifact.url, `${artifact.type}:code`);
   }
 
-  if (artifact.type === "stylesheet") {
-    return extractFromCss(artifact.content, artifact.url, "stylesheet:code");
-  }
-
   return [];
 }

package/lib/cli-args.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-import { providerTypeSchema } from "./provider";
+import { getProviderPreset, providerPresetSchema, providerTypeSchema } from "./provider";
 
 const rawCliArgsSchema = z.object({
   help: z.boolean().default(false),
@@ -13,6 +13,7 @@ const rawCliArgsSchema = z.object({
   url: z.string().url().optional(),
   output: z.string().min(1).optional(),
   providerType: providerTypeSchema.optional(),
+  providerPreset: providerPresetSchema.optional(),
   providerName: z.string().min(1).optional(),
   apiKey: z.string().min(1).optional(),
   baseURL: z.string().url().optional(),
@@ -20,11 +21,13 @@ const rawCliArgsSchema = z.object({
   contextSize: z.number().int().positive().optional(),
   maxPages: z.number().int().positive().optional(),
   maxArtifacts: z.number().int().positive().optional(),
+  maxDepth: z.number().int().nonnegative().optional(),
 });
 
 const cliConfigOverrideSchema = z
   .object({
     providerType: providerTypeSchema.optional(),
+    providerPreset: providerPresetSchema.optional(),
     providerName: z.string().min(1).optional(),
     apiKey: z.string().min(1).optional(),
     baseURL: z.string().url().optional(),
@@ -49,6 +52,7 @@ const optionMap = new Map<string, keyof CliArgs>([
   ["-u", "url"],
   ["--output", "output"],
   ["--provider-type", "providerType"],
+  ["--provider-preset", "providerPreset"],
   ["--provider-name", "providerName"],
   ["--api-key", "apiKey"],
   ["--base-url", "baseURL"],
@@ -56,10 +60,11 @@ const optionMap = new Map<string, keyof CliArgs>([
   ["--context-size", "contextSize"],
   ["--max-pages", "maxPages"],
   ["--max-artifacts", "maxArtifacts"],
+  ["--max-depth", "maxDepth"],
 ]);
 
 const booleanKeys = new Set<keyof CliArgs>(["help", "version", "headless", "reconfigure", "listModels", "localRag", "verboseAgents"]);
-const numberKeys = new Set<keyof CliArgs>(["contextSize", "maxPages", "maxArtifacts"]);
+const numberKeys = new Set<keyof CliArgs>(["contextSize", "maxPages", "maxArtifacts", "maxDepth"]);
 
 function normalizeValue(key: keyof CliArgs, value: string): unknown {
   if (numberKeys.has(key)) {
@@ -110,6 +115,13 @@ export function getConfigOverrides(args: CliArgs) {
   const overrides: Record<string, unknown> = {};
 
   if (args.providerType !== undefined) overrides.providerType = args.providerType;
+  if (args.providerPreset !== undefined) {
+    const preset = getProviderPreset(args.providerPreset);
+    overrides.providerType = "openai-compatible";
+    overrides.providerPreset = args.providerPreset;
+    overrides.providerName = args.providerName ?? preset.providerName;
+    overrides.baseURL = args.baseURL ?? preset.baseURL;
+  }
   if (args.providerName !== undefined) overrides.providerName = args.providerName;
   if (args.apiKey !== undefined) overrides.apiKey = args.apiKey;
   if (args.baseURL !== undefined) overrides.baseURL = args.baseURL;
@@ -132,9 +144,11 @@ export function renderHelpText(): string {
     "  --output <path>                 Write the report to a specific path",
     "  --max-pages <number>            Limit same-origin HTML pages to crawl",
     "  --max-artifacts <number>        Limit total downloaded artifacts",
+    "  --max-depth <number>            Limit crawl hop depth from the entry page",
     "",
     "Provider options:",
     "  --provider-type <type>          openai | openai-compatible",
+    "  --provider-preset <preset>      custom | blackbox | nvidia-nim | onlysq",
     "  --provider-name <name>          Display name for the provider",
     "  --api-key <key>                 Provider API key",
     "  --base-url <url>                Base URL for the provider",