@redstone-md/mapr 0.0.1-alpha

package/lib/artifacts.ts

@@ -0,0 +1,233 @@
+import * as cheerio from "cheerio";
+import { z } from "zod";
+
+export const artifactTypeSchema = z.enum([
+  "html",
+  "script",
+  "service-worker",
+  "worker",
+  "stylesheet",
+  "manifest",
+  "json",
+  "wasm",
+]);
+
+export const discoveredArtifactSchema = z.object({
+  url: z.string().url(),
+  type: artifactTypeSchema,
+  sizeBytes: z.number().int().nonnegative(),
+  content: z.string(),
+  discoveredFrom: z.string().min(1),
+});
+
+export const artifactCandidateSchema = z.object({
+  url: z.string().url(),
+  type: artifactTypeSchema,
+  discoveredFrom: z.string().min(1),
+});
+
+export type ArtifactType = z.infer<typeof artifactTypeSchema>;
+export type DiscoveredArtifact = z.infer<typeof discoveredArtifactSchema>;
+export type ArtifactCandidate = z.infer<typeof artifactCandidateSchema>;
+
+function makeCandidate(url: string, type: ArtifactType, discoveredFrom: string): ArtifactCandidate | null {
+  const parsed = artifactCandidateSchema.safeParse({ url, type, discoveredFrom });
+  return parsed.success ? parsed.data : null;
+}
+
+function addCandidate(
+  candidates: Map<string, ArtifactCandidate>,
+  candidate: ArtifactCandidate | null,
+  restrictToSameOrigin: boolean,
+  origin: string,
+): void {
+  if (!candidate) {
+    return;
+  }
+
+  if (restrictToSameOrigin && new URL(candidate.url).origin !== origin) {
+    return;
+  }
+
+  if (!candidates.has(candidate.url)) {
+    candidates.set(candidate.url, candidate);
+  }
+}
+
+function resolveCandidateUrl(reference: string, baseUrl: string): string | null {
+  if (!reference || reference.startsWith("data:") || reference.startsWith("blob:") || reference.startsWith("#")) {
+    return null;
+  }
+
+  try {
+    const absoluteUrl = new URL(reference, baseUrl).toString();
+    const parsed = z.string().url().safeParse(absoluteUrl);
+    return parsed.success ? parsed.data : null;
+  } catch {
+    return null;
+  }
+}
+
+function inferAssetTypeFromUrl(url: string, fallback: ArtifactType = "script"): ArtifactType {
+  const pathname = new URL(url).pathname.toLowerCase();
+
+  if (pathname.endsWith(".wasm")) {
+    return "wasm";
+  }
+
+  if (pathname.endsWith(".css")) {
+    return "stylesheet";
+  }
+
+  if (pathname.endsWith(".json")) {
+    return "json";
+  }
+
+  if (pathname.endsWith(".webmanifest") || pathname.endsWith("manifest.json")) {
+    return "manifest";
+  }
+
+  if (pathname.endsWith(".html") || pathname.endsWith(".htm")) {
+    return "html";
+  }
+
+  return fallback;
+}
+
+function extractFromJavaScript(source: string, baseUrl: string, discoveredFrom: string): ArtifactCandidate[] {
+  const candidates = new Map<string, ArtifactCandidate>();
+  const regexDefinitions: Array<{ regex: RegExp; type: ArtifactType }> = [
+    { regex: /(?:import|export)\s+(?:[^"'`]+?\s+from\s+)?["'`]([^"'`]+)["'`]/g, type: "script" },
+    { regex: /import\(\s*["'`]([^"'`]+)["'`]\s*\)/g, type: "script" },
+    { regex: /navigator\.serviceWorker\.register\(\s*(?:new\s+URL\(\s*)?["'`]([^"'`]+)["'`]/g, type: "service-worker" },
+    { regex: /new\s+(?:SharedWorker|Worker)\(\s*(?:new\s+URL\(\s*)?["'`]([^"'`]+)["'`]/g, type: "worker" },
+    { regex: /["'`]([^"'`]+\.wasm(?:\?[^"'`]*)?)["'`]/g, type: "wasm" },
+  ];
+
+  for (const definition of regexDefinitions) {
+    let match: RegExpExecArray | null;
+    while ((match = definition.regex.exec(source)) !== null) {
+      const resolvedUrl = resolveCandidateUrl(match[1] ?? "", baseUrl);
+      if (!resolvedUrl) {
+        continue;
+      }
+
+      addCandidate(
+        candidates,
+        makeCandidate(resolvedUrl, inferAssetTypeFromUrl(resolvedUrl, definition.type), discoveredFrom),
+        false,
+        new URL(baseUrl).origin,
+      );
+    }
+  }
+
+  return [...candidates.values()];
+}
+
+function extractFromCss(source: string, baseUrl: string, discoveredFrom: string): ArtifactCandidate[] {
+  const candidates = new Map<string, ArtifactCandidate>();
+  const regexDefinitions = [
+    /@import\s+(?:url\()?["']?([^"'()]+)["']?\)?/g,
+    /url\(\s*["']?([^"'()]+)["']?\s*\)/g,
+  ];
+
+  for (const regex of regexDefinitions) {
+    let match: RegExpExecArray | null;
+    while ((match = regex.exec(source)) !== null) {
+      const resolvedUrl = resolveCandidateUrl(match[1] ?? "", baseUrl);
+      if (!resolvedUrl) {
+        continue;
+      }
+
+      addCandidate(
+        candidates,
+        makeCandidate(resolvedUrl, inferAssetTypeFromUrl(resolvedUrl, "stylesheet"), discoveredFrom),
+        false,
+        new URL(baseUrl).origin,
+      );
+    }
+  }
+
+  return [...candidates.values()];
+}
+
+export function extractArtifactCandidates(html: string, pageUrl: string): ArtifactCandidate[] {
+  const $ = cheerio.load(html);
+  const candidates = new Map<string, ArtifactCandidate>();
+  const origin = new URL(pageUrl).origin;
+
+  $("script[src]").each((_, element) => {
+    const src = resolveCandidateUrl($(element).attr("src")?.trim() ?? "", pageUrl);
+    addCandidate(candidates, makeCandidate(src ?? "", "script", "html:script"), false, origin);
+  });
+
+  $("link[href]").each((_, element) => {
+    const href = resolveCandidateUrl($(element).attr("href")?.trim() ?? "", pageUrl);
+    if (!href) {
+      return;
+    }
+
+    const rel = ($(element).attr("rel") ?? "").toLowerCase();
+    const asValue = ($(element).attr("as") ?? "").toLowerCase();
+
+    if (rel.includes("manifest")) {
+      addCandidate(candidates, makeCandidate(href, "manifest", "html:manifest"), false, origin);
+      return;
+    }
+
+    if (rel.includes("stylesheet")) {
+      addCandidate(candidates, makeCandidate(href, "stylesheet", "html:stylesheet"), false, origin);
+      return;
+    }
+
+    if (rel.includes("modulepreload") || (rel.includes("preload") && asValue === "script")) {
+      addCandidate(candidates, makeCandidate(href, "script", "html:preload"), false, origin);
+    }
+  });
+
+  $("a[href]").each((_, element) => {
+    const href = resolveCandidateUrl($(element).attr("href")?.trim() ?? "", pageUrl);
+    if (!href) {
+      return;
+    }
+
+    const url = new URL(href);
+    const isSameOrigin = url.origin === origin;
+    const pathname = url.pathname.toLowerCase();
+    const looksLikePage =
+      pathname === "" ||
+      pathname.endsWith("/") ||
+      pathname.endsWith(".html") ||
+      pathname.endsWith(".htm") ||
+      !/\.[a-z0-9]+$/i.test(pathname);
+
+    if (isSameOrigin && looksLikePage) {
+      addCandidate(candidates, makeCandidate(href, "html", "html:anchor"), true, origin);
+    }
+  });
+
+  $("script:not([src])").each((_, element) => {
+    const inlineSource = $(element).html() ?? "";
+    for (const candidate of extractFromJavaScript(inlineSource, pageUrl, "html:inline-script")) {
+      addCandidate(candidates, candidate, false, origin);
+    }
+  });
+
+  return [...candidates.values()];
+}
+
+export function extractNestedCandidates(artifact: DiscoveredArtifact): ArtifactCandidate[] {
+  if (artifact.type === "html") {
+    return extractArtifactCandidates(artifact.content, artifact.url);
+  }
+
+  if (artifact.type === "script" || artifact.type === "service-worker" || artifact.type === "worker") {
+    return extractFromJavaScript(artifact.content, artifact.url, `${artifact.type}:code`);
+  }
+
+  if (artifact.type === "stylesheet") {
+    return extractFromCss(artifact.content, artifact.url, "stylesheet:code");
+  }
+
+  return [];
+}

package/lib/cli-args.ts

@@ -0,0 +1,152 @@
+import { z } from "zod";
+
+import { providerTypeSchema } from "./provider";
+
+const rawCliArgsSchema = z.object({
+  help: z.boolean().default(false),
+  version: z.boolean().default(false),
+  headless: z.boolean().default(false),
+  reconfigure: z.boolean().default(false),
+  listModels: z.boolean().default(false),
+  localRag: z.boolean().default(false),
+  verboseAgents: z.boolean().default(false),
+  url: z.string().url().optional(),
+  output: z.string().min(1).optional(),
+  providerType: providerTypeSchema.optional(),
+  providerName: z.string().min(1).optional(),
+  apiKey: z.string().min(1).optional(),
+  baseURL: z.string().url().optional(),
+  model: z.string().min(1).optional(),
+  contextSize: z.number().int().positive().optional(),
+  maxPages: z.number().int().positive().optional(),
+  maxArtifacts: z.number().int().positive().optional(),
+});
+
+const cliConfigOverrideSchema = z
+  .object({
+    providerType: providerTypeSchema.optional(),
+    providerName: z.string().min(1).optional(),
+    apiKey: z.string().min(1).optional(),
+    baseURL: z.string().url().optional(),
+    model: z.string().min(1).optional(),
+    modelContextSize: z.number().int().positive().optional(),
+  })
+  .strict();
+
+export type CliArgs = z.infer<typeof rawCliArgsSchema>;
+
+const optionMap = new Map<string, keyof CliArgs>([
+  ["--help", "help"],
+  ["-h", "help"],
+  ["--version", "version"],
+  ["-v", "version"],
+  ["--headless", "headless"],
+  ["--reconfigure", "reconfigure"],
+  ["--list-models", "listModels"],
+  ["--local-rag", "localRag"],
+  ["--verbose-agents", "verboseAgents"],
+  ["--url", "url"],
+  ["-u", "url"],
+  ["--output", "output"],
+  ["--provider-type", "providerType"],
+  ["--provider-name", "providerName"],
+  ["--api-key", "apiKey"],
+  ["--base-url", "baseURL"],
+  ["--model", "model"],
+  ["--context-size", "contextSize"],
+  ["--max-pages", "maxPages"],
+  ["--max-artifacts", "maxArtifacts"],
+]);
+
+const booleanKeys = new Set<keyof CliArgs>(["help", "version", "headless", "reconfigure", "listModels", "localRag", "verboseAgents"]);
+const numberKeys = new Set<keyof CliArgs>(["contextSize", "maxPages", "maxArtifacts"]);
+
+function normalizeValue(key: keyof CliArgs, value: string): unknown {
+  if (numberKeys.has(key)) {
+    return Number(value);
+  }
+
+  return value;
+}
+
+export function parseCliArgs(argv: string[]): CliArgs {
+  const accumulator: Record<string, unknown> = {};
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (!token) {
+      continue;
+    }
+    if (!token.startsWith("-")) {
+      throw new Error(`Unexpected positional argument: ${token}`);
+    }
+
+    const [rawKey, rawInlineValue] = token.includes("=") ? token.split(/=(.*)/s, 2) : [token, undefined];
+    const mappedKey = optionMap.get(rawKey);
+    if (!mappedKey) {
+      throw new Error(`Unknown argument: ${rawKey}`);
+    }
+
+    if (booleanKeys.has(mappedKey)) {
+      accumulator[mappedKey] = true;
+      continue;
+    }
+
+    const value = rawInlineValue ?? argv[index + 1];
+    if (!value || value.startsWith("-")) {
+      throw new Error(`Argument ${rawKey} requires a value.`);
+    }
+
+    accumulator[mappedKey] = normalizeValue(mappedKey, value);
+    if (rawInlineValue === undefined) {
+      index += 1;
+    }
+  }
+
+  return rawCliArgsSchema.parse(accumulator);
+}
+
+export function getConfigOverrides(args: CliArgs) {
+  const overrides: Record<string, unknown> = {};
+
+  if (args.providerType !== undefined) overrides.providerType = args.providerType;
+  if (args.providerName !== undefined) overrides.providerName = args.providerName;
+  if (args.apiKey !== undefined) overrides.apiKey = args.apiKey;
+  if (args.baseURL !== undefined) overrides.baseURL = args.baseURL;
+  if (args.model !== undefined) overrides.model = args.model;
+  if (args.contextSize !== undefined) overrides.modelContextSize = args.contextSize;
+
+  return cliConfigOverrideSchema.parse(overrides);
+}
+
+export function renderHelpText(): string {
+  return [
+    "Mapr",
+    "",
+    "Usage:",
+    "  mapr [options]",
+    "",
+    "Core options:",
+    "  --url, -u <url>                 Target URL to analyze",
+    "  --headless                      Disable prompts and require config from saved values or flags",
+    "  --output <path>                 Write the report to a specific path",
+    "  --max-pages <number>            Limit same-origin HTML pages to crawl",
+    "  --max-artifacts <number>        Limit total downloaded artifacts",
+    "",
+    "Provider options:",
+    "  --provider-type <type>          openai | openai-compatible",
+    "  --provider-name <name>          Display name for the provider",
+    "  --api-key <key>                 Provider API key",
+    "  --base-url <url>                Base URL for the provider",
+    "  --model <id>                    Model identifier",
+    "  --context-size <tokens>         Model context window, for example 128000 or 512000",
+    "  --list-models                   Fetch and print models using the resolved provider config",
+    "  --local-rag                     Enable local lexical RAG for oversized artifacts",
+    "  --reconfigure                   Force interactive provider reconfiguration",
+    "",
+    "Output and diagnostics:",
+    "  --verbose-agents                Print agent swarm completion events",
+    "  --help, -h                      Show help",
+    "  --version, -v                   Show version",
+  ].join("\n");
+}