@detiq/crawler 1.1.23 → 1.1.24

package/dist/extractors/branding-extractor.js

@@ -0,0 +1,104 @@
+async function extractBranding(page) {
+  try {
+    const result = await page.evaluate(() => {
+      const bodyStyle = window.getComputedStyle(document.body);
+      const isDark = window.matchMedia("(prefers-color-scheme: dark)").matches || document.documentElement.getAttribute("data-theme") === "dark" || document.body.classList.contains("dark");
+      const bg = bodyStyle.backgroundColor;
+      const text = bodyStyle.color;
+      const primaryBtn = document.querySelector(
+        'button[class*="primary"], .btn-primary, [class*="btn-primary"], [class*="button-primary"]'
+      );
+      const primaryColor = primaryBtn ? window.getComputedStyle(primaryBtn).backgroundColor : void 0;
+      const firstLink = document.querySelector("a");
+      const linkColor = firstLink ? window.getComputedStyle(firstLink).color : void 0;
+      const headingEl = document.querySelector("h1, h2");
+      const bodyEl = document.body;
+      const headingFont = headingEl ? window.getComputedStyle(headingEl).fontFamily : bodyStyle.fontFamily;
+      const bodyFont = bodyStyle.fontFamily;
+      const logoEl = document.querySelector(
+        'header img, img[alt*="logo" i], img[class*="logo" i], [class*="logo"] img, [id*="logo"] img'
+      );
+      const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+      const ogImageEl = document.querySelector('meta[property="og:image"]');
+      const titleEl = document.querySelector("title");
+      const descEl = document.querySelector('meta[name="description"]');
+      const themeColorEl = document.querySelector('meta[name="theme-color"]');
+      const hasBootstrap = !!document.querySelector('[class*="btn btn-"], .container-fluid, .navbar-expand');
+      const hasTailwind = !!document.querySelector('[class*="flex "], [class*="text-"], [class*="bg-"]');
+      const hasChakra = !!document.__chakra_factory__ || !!document.querySelector('[class*="chakra-"]');
+      const hasMUI = !!document.querySelector(".MuiButton-root, .MuiTypography-root");
+      const framework = hasBootstrap ? "Bootstrap" : hasTailwind ? "Tailwind CSS" : hasChakra ? "Chakra UI" : hasMUI ? "Material UI" : null;
+      return {
+        isDark,
+        bg,
+        text,
+        primaryColor,
+        linkColor,
+        headingFont,
+        bodyFont,
+        logo: logoEl?.src || logoEl?.getAttribute("src") || void 0,
+        favicon: faviconEl?.href || void 0,
+        ogImage: ogImageEl?.content || void 0,
+        title: titleEl?.textContent || void 0,
+        description: descEl?.content || void 0,
+        themeColor: themeColorEl?.content || void 0,
+        language: document.documentElement.lang || void 0,
+        framework
+      };
+    });
+    const fonts = [];
+    if (result.headingFont && result.headingFont !== result.bodyFont) {
+      fonts.push({ family: result.headingFont.split(",")[0].replace(/['"]/g, "").trim(), role: "heading" });
+    }
+    if (result.bodyFont) {
+      fonts.push({ family: result.bodyFont.split(",")[0].replace(/['"]/g, "").trim(), role: "body" });
+    }
+    const colorConfidence = result.bg || result.text || result.primaryColor ? 0.8 : 0.3;
+    const fontConfidence = fonts.length > 0 ? 0.7 : 0.2;
+    const imageConfidence = result.logo || result.ogImage ? 0.9 : 0.2;
+    return {
+      colorScheme: result.isDark ? "dark" : "light",
+      colors: {
+        background: result.bg || void 0,
+        textPrimary: result.text || void 0,
+        primary: result.primaryColor || void 0,
+        link: result.linkColor || void 0
+      },
+      fonts,
+      images: {
+        logo: result.logo,
+        favicon: result.favicon,
+        ogImage: result.ogImage
+      },
+      designSystem: {
+        framework: result.framework || void 0,
+        componentLibrary: void 0
+      },
+      metadata: {
+        title: result.title,
+        description: result.description,
+        themeColor: result.themeColor,
+        language: result.language
+      },
+      confidence: {
+        colors: colorConfidence,
+        fonts: fontConfidence,
+        images: imageConfidence,
+        overall: (colorConfidence + fontConfidence + imageConfidence) / 3
+      }
+    };
+  } catch (err) {
+    console.warn("[branding-extractor] Extraction failed:", err);
+    return {
+      colorScheme: "light",
+      colors: {},
+      fonts: [],
+      images: {},
+      confidence: { colors: 0, fonts: 0, images: 0, overall: 0 }
+    };
+  }
+}
+export {
+  extractBranding
+};
+//# sourceMappingURL=branding-extractor.js.map

package/dist/extractors/branding-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/branding-extractor.ts"],
+  "sourcesContent": ["import type { Page } from 'playwright';\n\nexport interface BrandingColors {\n  primary?: string;\n  accent?: string;\n  background?: string;\n  textPrimary?: string;\n  link?: string;\n}\n\nexport interface BrandingFont {\n  family: string;\n  role: 'heading' | 'body';\n}\n\nexport interface BrandingImages {\n  logo?: string;\n  favicon?: string;\n  ogImage?: string;\n}\n\nexport interface BrandingPersonality {\n  tone?: string;\n  energy?: 'high' | 'medium' | 'low';\n  targetAudience?: string;\n}\n\nexport interface BrandingResult {\n  colorScheme: 'light' | 'dark';\n  colors: BrandingColors;\n  fonts: BrandingFont[];\n  images: BrandingImages;\n  personality?: BrandingPersonality;\n  designSystem?: { framework?: string; componentLibrary?: string };\n  metadata?: { title?: string; description?: string; themeColor?: string; language?: string };\n  confidence: { colors: number; fonts: number; images: number; overall: number };\n}\n\nexport async function extractBranding(page: Page): Promise<BrandingResult> {\n  try {\n    const result = await page.evaluate(() => {\n      const bodyStyle = window.getComputedStyle(document.body);\n      const isDark = window.matchMedia('(prefers-color-scheme: dark)').matches ||\n        document.documentElement.getAttribute('data-theme') === 'dark' ||\n        document.body.classList.contains('dark');\n\n      // Extract colors from computed styles\n      const bg = bodyStyle.backgroundColor;\n      const text = bodyStyle.color;\n\n      // Find primary button color\n      const primaryBtn = document.querySelector(\n        'button[class*=\"primary\"], .btn-primary, [class*=\"btn-primary\"], [class*=\"button-primary\"]'\n      );\n      const primaryColor = primaryBtn ? window.getComputedStyle(primaryBtn).backgroundColor : undefined;\n\n      // Find link color\n      const firstLink = document.querySelector('a');\n      const linkColor = firstLink ? window.getComputedStyle(firstLink).color : undefined;\n\n      // Extract font families\n      const headingEl = document.querySelector('h1, h2');\n      const bodyEl = document.body;\n      const headingFont = headingEl ? window.getComputedStyle(headingEl).fontFamily : bodyStyle.fontFamily;\n      const bodyFont = bodyStyle.fontFamily;\n\n      // Extract images\n      const logoEl = document.querySelector(\n        'header img, img[alt*=\"logo\" i], img[class*=\"logo\" i], [class*=\"logo\"] img, [id*=\"logo\"] img'\n      ) as HTMLImageElement | null;\n      const faviconEl = document.querySelector('link[rel=\"icon\"], link[rel=\"shortcut icon\"]') as HTMLLinkElement | null;\n      const ogImageEl = document.querySelector('meta[property=\"og:image\"]') as HTMLMetaElement | null;\n\n      // Extract metadata\n      const titleEl = document.querySelector('title');\n      const descEl = document.querySelector('meta[name=\"description\"]') as HTMLMetaElement | null;\n      const themeColorEl = document.querySelector('meta[name=\"theme-color\"]') as HTMLMetaElement | null;\n\n      // Detect design framework\n      const hasBootstrap = !!document.querySelector('[class*=\"btn btn-\"], .container-fluid, .navbar-expand');\n      const hasTailwind = !!document.querySelector('[class*=\"flex \"], [class*=\"text-\"], [class*=\"bg-\"]');\n      const hasChakra = !!(document as any).__chakra_factory__ || !!document.querySelector('[class*=\"chakra-\"]');\n      const hasMUI = !!document.querySelector('.MuiButton-root, .MuiTypography-root');\n\n      const framework = hasBootstrap ? 'Bootstrap' : hasTailwind ? 'Tailwind CSS' : hasChakra ? 'Chakra UI' : hasMUI ? 'Material UI' : null;\n\n      return {\n        isDark,\n        bg,\n        text,\n        primaryColor,\n        linkColor,\n        headingFont,\n        bodyFont,\n        logo: logoEl?.src || logoEl?.getAttribute('src') || undefined,\n        favicon: faviconEl?.href || undefined,\n        ogImage: ogImageEl?.content || undefined,\n        title: titleEl?.textContent || undefined,\n        description: descEl?.content || undefined,\n        themeColor: themeColorEl?.content || undefined,\n        language: document.documentElement.lang || undefined,\n        framework,\n      };\n    });\n\n    const fonts: BrandingFont[] = [];\n    if (result.headingFont && result.headingFont !== result.bodyFont) {\n      fonts.push({ family: result.headingFont.split(',')[0].replace(/['\"]/g, '').trim(), role: 'heading' });\n    }\n    if (result.bodyFont) {\n      fonts.push({ family: result.bodyFont.split(',')[0].replace(/['\"]/g, '').trim(), role: 'body' });\n    }\n\n    const colorConfidence = (result.bg || result.text || result.primaryColor) ? 0.8 : 0.3;\n    const fontConfidence = fonts.length > 0 ? 0.7 : 0.2;\n    const imageConfidence = (result.logo || result.ogImage) ? 0.9 : 0.2;\n\n    return {\n      colorScheme: result.isDark ? 'dark' : 'light',\n      colors: {\n        background: result.bg || undefined,\n        textPrimary: result.text || undefined,\n        primary: result.primaryColor || undefined,\n        link: result.linkColor || undefined,\n      },\n      fonts,\n      images: {\n        logo: result.logo,\n        favicon: result.favicon,\n        ogImage: result.ogImage,\n      },\n      designSystem: {\n        framework: result.framework || undefined,\n        componentLibrary: undefined,\n      },\n      metadata: {\n        title: result.title,\n        description: result.description,\n        themeColor: result.themeColor,\n        language: result.language,\n      },\n      confidence: {\n        colors: colorConfidence,\n        fonts: fontConfidence,\n        images: imageConfidence,\n        overall: (colorConfidence + fontConfidence + imageConfidence) / 3,\n      },\n    };\n  } catch (err) {\n    console.warn('[branding-extractor] Extraction failed:', err);\n    return {\n      colorScheme: 'light',\n      colors: {},\n      fonts: [],\n      images: {},\n      confidence: { colors: 0, fonts: 0, images: 0, overall: 0 },\n    };\n  }\n}\n"],
+  "mappings": "AAsCA,eAAsB,gBAAgB,MAAqC;AACzE,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,MAAM;AACvC,YAAM,YAAY,OAAO,iBAAiB,SAAS,IAAI;AACvD,YAAM,SAAS,OAAO,WAAW,8BAA8B,EAAE,WAC/D,SAAS,gBAAgB,aAAa,YAAY,MAAM,UACxD,SAAS,KAAK,UAAU,SAAS,MAAM;AAGzC,YAAM,KAAK,UAAU;AACrB,YAAM,OAAO,UAAU;AAGvB,YAAM,aAAa,SAAS;AAAA,QAC1B;AAAA,MACF;AACA,YAAM,eAAe,aAAa,OAAO,iBAAiB,UAAU,EAAE,kBAAkB;AAGxF,YAAM,YAAY,SAAS,cAAc,GAAG;AAC5C,YAAM,YAAY,YAAY,OAAO,iBAAiB,SAAS,EAAE,QAAQ;AAGzE,YAAM,YAAY,SAAS,cAAc,QAAQ;AACjD,YAAM,SAAS,SAAS;AACxB,YAAM,cAAc,YAAY,OAAO,iBAAiB,SAAS,EAAE,aAAa,UAAU;AAC1F,YAAM,WAAW,UAAU;AAG3B,YAAM,SAAS,SAAS;AAAA,QACtB;AAAA,MACF;AACA,YAAM,YAAY,SAAS,cAAc,6CAA6C;AACtF,YAAM,YAAY,SAAS,cAAc,2BAA2B;AAGpE,YAAM,UAAU,SAAS,cAAc,OAAO;AAC9C,YAAM,SAAS,SAAS,cAAc,0BAA0B;AAChE,YAAM,eAAe,SAAS,cAAc,0BAA0B;AAGtE,YAAM,eAAe,CAAC,CAAC,SAAS,cAAc,uDAAuD;AACrG,YAAM,cAAc,CAAC,CAAC,SAAS,cAAc,oDAAoD;AACjG,YAAM,YAAY,CAAC,CAAE,SAAiB,sBAAsB,CAAC,CAAC,SAAS,cAAc,oBAAoB;AACzG,YAAM,SAAS,CAAC,CAAC,SAAS,cAAc,sCAAsC;AAE9E,YAAM,YAAY,eAAe,cAAc,cAAc,iBAAiB,YAAY,cAAc,SAAS,gBAAgB;AAEjI,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM,QAAQ,OAAO,QAAQ,aAAa,KAAK,KAAK;AAAA,QACpD,SAAS,WAAW,QAAQ;AAAA,QAC5B,SAAS,WAAW,WAAW;AAAA,QAC/B,OAAO,SAAS,eAAe;AAAA,QAC/B,aAAa,QAAQ,WAAW;AAAA,QAChC,YAAY,cAAc,WAAW;AAAA,QACrC,UAAU,SAAS,gBAAgB,QAAQ;AAAA,QAC3C;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,QAAwB,CAAC;AAC/B,QAAI,OAAO,eAAe,OAAO,gBAAgB,OAAO,UAAU;AAChE,YAAM,KAAK,EAAE,QAAQ,OAAO,YAAY,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,SAAS,EAAE,EAAE,KAAK,GAAG,MAAM,UAAU,CAAC;AAAA,IACtG;AACA,QAAI,OAAO,UAAU;AACnB,YAAM,KAAK,EAAE,QAAQ,OAAO,SAAS,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,SAAS,EAAE,EAAE,KAAK,GAAG,MAAM,OAAO,CAAC;AAAA,IAChG;AAEA,UAAM,kBAAmB,OAAO,MAAM,OAAO,QAAQ,OAAO,eAAgB,MAAM;AAClF,UAAM,iBAAiB,MAAM,SAAS,IAAI,MAAM;AAChD,UAAM,kBAAmB,OAAO,QAAQ,OAAO,UAAW,MAAM;AAEhE,WAAO;AAAA,MACL,aAAa,OAAO,SAAS,SAAS;AAAA,MACtC,QAAQ;AAAA,QACN,YAAY,OAAO,MAAM;AAAA,QACzB,aAAa,OAAO,QAAQ;AAAA,QAC5B,SAAS,OAAO,gBAAgB;AAAA,QAChC,MAAM,OAAO,aAAa;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,QACN,MAAM,OAAO;AAAA,QACb,SAAS,OAAO;AAAA,QAChB,SAAS,OAAO;AAAA,MAClB;AAAA,MACA,cAAc;AAAA,QACZ,WAAW,OAAO,aAAa;AAAA,QAC/B,kBAAkB;AAAA,MACpB;AAAA,MACA,UAAU;AAAA,QACR,OAAO,OAAO;AAAA,QACd,aAAa,OAAO;AAAA,QACpB,YAAY,OAAO;AAAA,QACnB,UAAU,OAAO;AAAA,MACnB;AAAA,MACA,YAAY;AAAA,QACV,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,UAAU,kBAAkB,iBAAiB,mBAAmB;AAAA,MAClE;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,KAAK,2CAA2C,GAAG;AAC3D,WAAO;AAAA,MACL,aAAa;AAAA,MACb,QAAQ,CAAC;AAAA,MACT,OAAO,CAAC;AAAA,MACR,QAAQ,CAAC;AAAA,MACT,YAAY,EAAE,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,EAAE;AAAA,IAC3D;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/extractors/summary-extractor.js

@@ -0,0 +1,36 @@
+async function extractSummary(markdown, options) {
+  const input = markdown.slice(0, 8e3);
+  try {
+    const { ask } = await import("@detiq/agents").catch(() => ({ ask: null }));
+    if (!ask) {
+      return makeTruncated(input);
+    }
+    const summary = await ask(
+      "Summarize the following web page content in 2-4 concise sentences. Focus on the main purpose, key features, and primary value proposition. Return only the summary text, no preamble.",
+      input,
+      options?.model ? { provider: "anthropic", model: options.model } : void 0,
+      options?.maxTokens ?? 300
+    );
+    return {
+      summary: summary.trim(),
+      wordCount: markdown.split(/\s+/).length,
+      method: "llm",
+      model: options?.model
+    };
+  } catch (err) {
+    console.warn("[summary-extractor] LLM call failed, using truncated fallback:", err);
+    return makeTruncated(input);
+  }
+}
+function makeTruncated(input) {
+  const excerpt = input.slice(0, 500).trim();
+  return {
+    summary: excerpt + (input.length > 500 ? "..." : ""),
+    wordCount: excerpt.split(/\s+/).length,
+    method: "truncated"
+  };
+}
+export {
+  extractSummary
+};
+//# sourceMappingURL=summary-extractor.js.map

package/dist/extractors/summary-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/summary-extractor.ts"],
+  "sourcesContent": ["export interface SummaryResult {\n  summary: string;\n  wordCount: number;\n  method: 'llm' | 'truncated';\n  model?: string;\n}\n\nexport async function extractSummary(\n  markdown: string,\n  options?: { model?: string; maxTokens?: number },\n): Promise<SummaryResult> {\n  // Truncate to 8000 chars to stay within token budget\n  const input = markdown.slice(0, 8000);\n\n  try {\n    // Dynamic import to avoid hard dependency if agents package unavailable\n    const { ask } = await import('@detiq/agents').catch(() => ({ ask: null as any }));\n\n    if (!ask) {\n      return makeTruncated(input);\n    }\n\n    const summary: string = await ask(\n      'Summarize the following web page content in 2-4 concise sentences. Focus on the main purpose, key features, and primary value proposition. Return only the summary text, no preamble.',\n      input,\n      options?.model ? { provider: 'anthropic', model: options.model } : undefined,\n      options?.maxTokens ?? 300,\n    );\n\n    return {\n      summary: summary.trim(),\n      wordCount: markdown.split(/\\s+/).length,\n      method: 'llm',\n      model: options?.model,\n    };\n  } catch (err) {\n    console.warn('[summary-extractor] LLM call failed, using truncated fallback:', err);\n    return makeTruncated(input);\n  }\n}\n\nfunction makeTruncated(input: string): SummaryResult {\n  const excerpt = input.slice(0, 500).trim();\n  return {\n    summary: excerpt + (input.length > 500 ? '...' : ''),\n    wordCount: excerpt.split(/\\s+/).length,\n    method: 'truncated',\n  };\n}\n"],
+  "mappings": "AAOA,eAAsB,eACpB,UACA,SACwB;AAExB,QAAM,QAAQ,SAAS,MAAM,GAAG,GAAI;AAEpC,MAAI;AAEF,UAAM,EAAE,IAAI,IAAI,MAAM,OAAO,eAAe,EAAE,MAAM,OAAO,EAAE,KAAK,KAAY,EAAE;AAEhF,QAAI,CAAC,KAAK;AACR,aAAO,cAAc,KAAK;AAAA,IAC5B;AAEA,UAAM,UAAkB,MAAM;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,EAAE,UAAU,aAAa,OAAO,QAAQ,MAAM,IAAI;AAAA,MACnE,SAAS,aAAa;AAAA,IACxB;AAEA,WAAO;AAAA,MACL,SAAS,QAAQ,KAAK;AAAA,MACtB,WAAW,SAAS,MAAM,KAAK,EAAE;AAAA,MACjC,QAAQ;AAAA,MACR,OAAO,SAAS;AAAA,IAClB;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,KAAK,kEAAkE,GAAG;AAClF,WAAO,cAAc,KAAK;AAAA,EAC5B;AACF;AAEA,SAAS,cAAc,OAA8B;AACnD,QAAM,UAAU,MAAM,MAAM,GAAG,GAAG,EAAE,KAAK;AACzC,SAAO;AAAA,IACL,SAAS,WAAW,MAAM,SAAS,MAAM,QAAQ;AAAA,IACjD,WAAW,QAAQ,MAAM,KAAK,EAAE;AAAA,IAChC,QAAQ;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/frontier.js

@@ -1,3 +1,14 @@
+function matchesPattern(url, patterns) {
+  try {
+    const path = new URL(url).pathname;
+    return patterns.some((pattern) => {
+      const regex = new RegExp("^" + pattern.replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*") + "$");
+      return regex.test(path);
+    });
+  } catch {
+    return false;
+  }
+}
 function scoreUrl(url, keywords) {
   try {
     const u = new URL(url);
@@ -21,7 +32,9 @@ class CrawlFrontier {
   strategy;
   keywords;
   urlFilter;
-  constructor(strategy, seedUrls, keywords = [], urlFilter) {
+  includePatterns;
+  excludePatterns;
+  constructor(strategy, seedUrls, keywords = [], urlFilter, includePatterns, excludePatterns) {
     this.strategy = strategy;
     this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
     this.urlFilter = urlFilter ? (() => {
@@ -31,6 +44,8 @@ class CrawlFrontier {
         return null;
       }
     })() : null;
+    this.includePatterns = includePatterns ?? [];
+    this.excludePatterns = excludePatterns ?? [];
     for (const url of seedUrls) this.push(url);
   }
   get length() {
@@ -43,6 +58,8 @@ class CrawlFrontier {
    */
   push(url, pushOpts) {
     if (this.urlFilter && !this.urlFilter.test(url)) return;
+    if (this.includePatterns.length && !matchesPattern(url, this.includePatterns)) return;
+    if (this.excludePatterns.length && matchesPattern(url, this.excludePatterns)) return;
     const sparseBoost = this.strategy === "BEST_FIRST" && pushOpts?.parentElements !== void 0 && pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? "") || 5) ? 1 : 0;
     this.items.push({ url, score: this.strategy === "BEST_FIRST" ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
   }
@@ -80,6 +97,7 @@ class CrawlFrontier {
   }
 }
 export {
-  CrawlFrontier
+  CrawlFrontier,
+  matchesPattern
 };
 //# sourceMappingURL=frontier.js.map

package/dist/frontier.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/frontier.ts"],
-  "sourcesContent": ["/**\n * Configurable crawl traversal strategy \u2014 matches crawl4AI's BFS/DFS/\n * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded\n * FIFO queue, i.e. always BFS).\n *\n * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it\n * drops into crawler.ts's existing queue call sites with a rename, not a\n * rewrite \u2014 BFS's `shift()` behavior is byte-for-byte the same as the plain\n * array it replaces, so BFS (the default) has zero behavior change.\n */\n\nexport type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';\n\ninterface FrontierEntry {\n  url: string;\n  score: number;\n}\n\n/**\n * Best-First scoring heuristic: shallower paths score higher (breadth-like\n * bias toward top-level sections before deep sub-pages), boosted further by\n * any user-supplied keyword appearing in the URL. Simple and transparent by\n * design \u2014 crawl4AI's own default scorer is a similarly simple keyword-\n * relevance heuristic, not a learned model.\n */\nfunction scoreUrl(url: string, keywords: string[]): number {\n  try {\n    const u = new URL(url);\n    const segments = u.pathname.split('/').filter(Boolean);\n    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;\n    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;\n    let score = -segments.length * depthPenaltyWeight;\n    if (keywords.length > 0) {\n      const lower = url.toLowerCase();\n      for (const kw of keywords) {\n        if (kw && lower.includes(kw)) score += keywordBoostWeight;\n      }\n    }\n    return score;\n  } catch {\n    return 0;\n  }\n}\n\nexport class CrawlFrontier {\n  private items: FrontierEntry[] = [];\n  private readonly strategy: TraversalStrategy;\n  private readonly keywords: string[];\n  private readonly urlFilter: RegExp | null;\n\n  constructor(strategy: TraversalStrategy, seedUrls: string[], keywords: string[] = [], urlFilter?: string) {\n    this.strategy = strategy;\n    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n    this.urlFilter = urlFilter\n      ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()\n      : null;\n    for (const url of seedUrls) this.push(url);\n  }\n\n  get length(): number {\n    return this.items.length;\n  }\n\n  /**\n   * P3b: sparse-parent boost \u2014 if the parent page had fewer than 5 interactive\n   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so\n   * the frontier revisits sparse branches before deep-linked content.\n   */\n  push(url: string, pushOpts?: { parentElements?: number }): void {\n    if (this.urlFilter && !this.urlFilter.test(url)) return;\n    const sparseBoost =\n      this.strategy === 'BEST_FIRST' &&\n      pushOpts?.parentElements !== undefined &&\n      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)\n        ? 1\n        : 0;\n    this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });\n  }\n\n  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */\n  shift(): string | undefined {\n    if (this.items.length === 0) return undefined;\n    switch (this.strategy) {\n      case 'DFS':\n        return this.items.pop()!.url;\n      case 'BEST_FIRST': {\n        let bestIdx = 0;\n        for (let i = 1; i < this.items.length; i++) {\n          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;\n        }\n        return this.items.splice(bestIdx, 1)[0].url;\n      }\n      case 'BFS':\n      default:\n        return this.items.shift()!.url;\n    }\n  }\n\n  filter(predicate: (url: string) => boolean): string[] {\n    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);\n  }\n\n  /** Remove queued URLs not matching the regex pattern. Returns count removed. */\n  filterByPattern(pattern: string): number {\n    try {\n      const re = new RegExp(pattern, 'i');\n      const before = this.items.length;\n      this.items = this.items.filter((e) => re.test(e.url));\n      return before - this.items.length;\n    } catch {\n      return 0;\n    }\n  }\n}\n"],
-  "mappings": "AAyBA,SAAS,SAAS,KAAa,UAA4B;AACzD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,WAAW,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AACrD,UAAM,qBAAqB,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AAC3F,UAAM,qBAAqB,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AAC3F,QAAI,QAAQ,CAAC,SAAS,SAAS;AAC/B,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,QAAQ,IAAI,YAAY;AAC9B,iBAAW,MAAM,UAAU;AACzB,YAAI,MAAM,MAAM,SAAS,EAAE,EAAG,UAAS;AAAA,MACzC;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAc;AAAA,EACjB,QAAyB,CAAC;AAAA,EACjB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAA6B,UAAoB,WAAqB,CAAC,GAAG,WAAoB;AACxG,SAAK,WAAW;AAChB,SAAK,WAAW,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACnE,SAAK,YAAY,aACZ,MAAM;AAAE,UAAI;AAAE,eAAO,IAAI,OAAO,WAAW,GAAG;AAAA,MAAG,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IAAE,GAAG,IAC9E;AACJ,eAAW,OAAO,SAAU,MAAK,KAAK,GAAG;AAAA,EAC3C;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KAAK,KAAa,UAA8C;AAC9D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,KAAK,GAAG,EAAG;AACjD,UAAM,cACJ,KAAK,aAAa,gBAClB,UAAU,mBAAmB,UAC7B,SAAS,kBAAkB,SAAS,QAAQ,IAAI,4CAA4C,EAAE,KAAK,KAC/F,IACA;AACN,SAAK,MAAM,KAAK,EAAE,KAAK,OAAO,KAAK,aAAa,eAAe,SAAS,KAAK,KAAK,QAAQ,IAAI,cAAc,EAAE,CAAC;AAAA,EACjH;AAAA;AAAA,EAGA,QAA4B;AAC1B,QAAI,KAAK,MAAM,WAAW,EAAG,QAAO;AACpC,YAAQ,KAAK,UAAU;AAAA,MACrB,KAAK;AACH,eAAO,KAAK,MAAM,IAAI,EAAG;AAAA,MAC3B,KAAK,cAAc;AACjB,YAAI,UAAU;AACd,iBAAS,IAAI,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AAC1C,cAAI,KAAK,MAAM,CAAC,EAAE,QAAQ,KAAK,MAAM,OAAO,EAAE,MAAO,WAAU;AAAA,QACjE;AACA,eAAO,KAAK,MAAM,OAAO,SAAS,CAAC,EAAE,CAAC,EAAE;AAAA,MAC1C;AAAA,MACA,KAAK;AAAA,MACL;AACE,eAAO,KAAK,MAAM,MAAM,EAAG;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,OAAO,WAA+C;AACpD,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,UAAU,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG;AAAA,EACpE;AAAA;AAAA,EAGA,gBAAgB,SAAyB;AACvC,QAAI;AACF,YAAM,KAAK,IAAI,OAAO,SAAS,GAAG;AAClC,YAAM,SAAS,KAAK,MAAM;AAC1B,WAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC;AACpD,aAAO,SAAS,KAAK,MAAM;AAAA,IAC7B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
+  "sourcesContent": ["/**\n * Configurable crawl traversal strategy \u2014 matches crawl4AI's BFS/DFS/\n * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded\n * FIFO queue, i.e. always BFS).\n *\n * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it\n * drops into crawler.ts's existing queue call sites with a rename, not a\n * rewrite \u2014 BFS's `shift()` behavior is byte-for-byte the same as the plain\n * array it replaces, so BFS (the default) has zero behavior change.\n */\n\nexport type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';\n\n/**\n * Simple glob-like pattern matching against a URL's pathname.\n * Supports `*` (matches any single path segment characters, not `/`)\n * and `**` (matches anything including `/`).\n */\nexport function matchesPattern(url: string, patterns: string[]): boolean {\n  try {\n    const path = new URL(url).pathname;\n    return patterns.some(pattern => {\n      const regex = new RegExp('^' + pattern.replace(/\\*\\*/g, '.*').replace(/\\*/g, '[^/]*') + '$');\n      return regex.test(path);\n    });\n  } catch {\n    return false;\n  }\n}\n\ninterface FrontierEntry {\n  url: string;\n  score: number;\n}\n\n/**\n * Best-First scoring heuristic: shallower paths score higher (breadth-like\n * bias toward top-level sections before deep sub-pages), boosted further by\n * any user-supplied keyword appearing in the URL. Simple and transparent by\n * design \u2014 crawl4AI's own default scorer is a similarly simple keyword-\n * relevance heuristic, not a learned model.\n */\nfunction scoreUrl(url: string, keywords: string[]): number {\n  try {\n    const u = new URL(url);\n    const segments = u.pathname.split('/').filter(Boolean);\n    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;\n    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;\n    let score = -segments.length * depthPenaltyWeight;\n    if (keywords.length > 0) {\n      const lower = url.toLowerCase();\n      for (const kw of keywords) {\n        if (kw && lower.includes(kw)) score += keywordBoostWeight;\n      }\n    }\n    return score;\n  } catch {\n    return 0;\n  }\n}\n\nexport class CrawlFrontier {\n  private items: FrontierEntry[] = [];\n  private readonly strategy: TraversalStrategy;\n  private readonly keywords: string[];\n  private readonly urlFilter: RegExp | null;\n  private readonly includePatterns: string[];\n  private readonly excludePatterns: string[];\n\n  constructor(\n    strategy: TraversalStrategy,\n    seedUrls: string[],\n    keywords: string[] = [],\n    urlFilter?: string,\n    includePatterns?: string[],\n    excludePatterns?: string[],\n  ) {\n    this.strategy = strategy;\n    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n    this.urlFilter = urlFilter\n      ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()\n      : null;\n    this.includePatterns = includePatterns ?? [];\n    this.excludePatterns = excludePatterns ?? [];\n    for (const url of seedUrls) this.push(url);\n  }\n\n  get length(): number {\n    return this.items.length;\n  }\n\n  /**\n   * P3b: sparse-parent boost \u2014 if the parent page had fewer than 5 interactive\n   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so\n   * the frontier revisits sparse branches before deep-linked content.\n   */\n  push(url: string, pushOpts?: { parentElements?: number }): void {\n    if (this.urlFilter && !this.urlFilter.test(url)) return;\n    if (this.includePatterns.length && !matchesPattern(url, this.includePatterns)) return;\n    if (this.excludePatterns.length && matchesPattern(url, this.excludePatterns)) return;\n    const sparseBoost =\n      this.strategy === 'BEST_FIRST' &&\n      pushOpts?.parentElements !== undefined &&\n      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)\n        ? 1\n        : 0;\n    this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });\n  }\n\n  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */\n  shift(): string | undefined {\n    if (this.items.length === 0) return undefined;\n    switch (this.strategy) {\n      case 'DFS':\n        return this.items.pop()!.url;\n      case 'BEST_FIRST': {\n        let bestIdx = 0;\n        for (let i = 1; i < this.items.length; i++) {\n          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;\n        }\n        return this.items.splice(bestIdx, 1)[0].url;\n      }\n      case 'BFS':\n      default:\n        return this.items.shift()!.url;\n    }\n  }\n\n  filter(predicate: (url: string) => boolean): string[] {\n    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);\n  }\n\n  /** Remove queued URLs not matching the regex pattern. Returns count removed. */\n  filterByPattern(pattern: string): number {\n    try {\n      const re = new RegExp(pattern, 'i');\n      const before = this.items.length;\n      this.items = this.items.filter((e) => re.test(e.url));\n      return before - this.items.length;\n    } catch {\n      return 0;\n    }\n  }\n}\n"],
+  "mappings": "AAkBO,SAAS,eAAe,KAAa,UAA6B;AACvE,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,GAAG,EAAE;AAC1B,WAAO,SAAS,KAAK,aAAW;AAC9B,YAAM,QAAQ,IAAI,OAAO,MAAM,QAAQ,QAAQ,SAAS,IAAI,EAAE,QAAQ,OAAO,OAAO,IAAI,GAAG;AAC3F,aAAO,MAAM,KAAK,IAAI;AAAA,IACxB,CAAC;AAAA,EACH,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAcA,SAAS,SAAS,KAAa,UAA4B;AACzD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,WAAW,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AACrD,UAAM,qBAAqB,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AAC3F,UAAM,qBAAqB,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AAC3F,QAAI,QAAQ,CAAC,SAAS,SAAS;AAC/B,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,QAAQ,IAAI,YAAY;AAC9B,iBAAW,MAAM,UAAU;AACzB,YAAI,MAAM,MAAM,SAAS,EAAE,EAAG,UAAS;AAAA,MACzC;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAc;AAAA,EACjB,QAAyB,CAAC;AAAA,EACjB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YACE,UACA,UACA,WAAqB,CAAC,GACtB,WACA,iBACA,iBACA;AACA,SAAK,WAAW;AAChB,SAAK,WAAW,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACnE,SAAK,YAAY,aACZ,MAAM;AAAE,UAAI;AAAE,eAAO,IAAI,OAAO,WAAW,GAAG;AAAA,MAAG,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IAAE,GAAG,IAC9E;AACJ,SAAK,kBAAkB,mBAAmB,CAAC;AAC3C,SAAK,kBAAkB,mBAAmB,CAAC;AAC3C,eAAW,OAAO,SAAU,MAAK,KAAK,GAAG;AAAA,EAC3C;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KAAK,KAAa,UAA8C;AAC9D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,KAAK,GAAG,EAAG;AACjD,QAAI,KAAK,gBAAgB,UAAU,CAAC,eAAe,KAAK,KAAK,eAAe,EAAG;AAC/E,QAAI,KAAK,gBAAgB,UAAU,eAAe,KAAK,KAAK,eAAe,EAAG;AAC9E,UAAM,cACJ,KAAK,aAAa,gBAClB,UAAU,mBAAmB,UAC7B,SAAS,kBAAkB,SAAS,QAAQ,IAAI,4CAA4C,EAAE,KAAK,KAC/F,IACA;AACN,SAAK,MAAM,KAAK,EAAE,KAAK,OAAO,KAAK,aAAa,eAAe,SAAS,KAAK,KAAK,QAAQ,IAAI,cAAc,EAAE,CAAC;AAAA,EACjH;AAAA;AAAA,EAGA,QAA4B;AAC1B,QAAI,KAAK,MAAM,WAAW,EAAG,QAAO;AACpC,YAAQ,KAAK,UAAU;AAAA,MACrB,KAAK;AACH,eAAO,KAAK,MAAM,IAAI,EAAG;AAAA,MAC3B,KAAK,cAAc;AACjB,YAAI,UAAU;AACd,iBAAS,IAAI,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AAC1C,cAAI,KAAK,MAAM,CAAC,EAAE,QAAQ,KAAK,MAAM,OAAO,EAAE,MAAO,WAAU;AAAA,QACjE;AACA,eAAO,KAAK,MAAM,OAAO,SAAS,CAAC,EAAE,CAAC,EAAE;AAAA,MAC1C;AAAA,MACA,KAAK;AAAA,MACL;AACE,eAAO,KAAK,MAAM,MAAM,EAAG;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,OAAO,WAA+C;AACpD,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,UAAU,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG;AAAA,EACpE;AAAA;AAAA,EAGA,gBAAgB,SAAyB;AACvC,QAAI;AACF,YAAM,KAAK,IAAI,OAAO,SAAS,GAAG;AAClC,YAAM,SAAS,KAAK,MAAM;AAC1B,WAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC;AACpD,aAAO,SAAS,KAAK,MAAM;AAAA,IAC7B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/incremental-diff.js

@@ -50,10 +50,86 @@ function partitionByChange(urls, priorHashes) {
   }
   return { toProcess, toSkip };
 }
+function computeTextDiffs(previous, current) {
+  const prevLines = previous.split("\n");
+  const currLines = current.split("\n");
+  const changes = [];
+  const prevSet = new Set(prevLines);
+  const currSet = new Set(currLines);
+  let lineNum = 0;
+  for (const line of prevLines) {
+    if (!currSet.has(line)) {
+      changes.push({ type: "removed", line: lineNum, content: line });
+    }
+    lineNum++;
+  }
+  lineNum = 0;
+  for (const line of currLines) {
+    if (!prevSet.has(line)) {
+      changes.push({ type: "added", line: lineNum, content: line });
+    }
+    lineNum++;
+  }
+  return changes;
+}
+function computeSetDiffs(previous, current) {
+  const prevSet = new Set(previous);
+  const currSet = new Set(current);
+  return {
+    added: current.filter((u) => !prevSet.has(u)),
+    removed: previous.filter((u) => !currSet.has(u))
+  };
+}
+function flattenJson(obj, prefix = "") {
+  if (typeof obj !== "object" || obj === null) {
+    return { [prefix]: obj };
+  }
+  const result = {};
+  for (const [k, v] of Object.entries(obj)) {
+    const path = prefix ? `${prefix}.${k}` : k;
+    if (typeof v === "object" && v !== null && !Array.isArray(v)) {
+      Object.assign(result, flattenJson(v, path));
+    } else {
+      result[path] = v;
+    }
+  }
+  return result;
+}
+function computeJsonDiffs(previous, current) {
+  const prevFlat = flattenJson(previous);
+  const currFlat = flattenJson(current);
+  const allKeys = /* @__PURE__ */ new Set([...Object.keys(prevFlat), ...Object.keys(currFlat)]);
+  const changes = [];
+  for (const path of allKeys) {
+    const oldVal = prevFlat[path];
+    const newVal = currFlat[path];
+    if (JSON.stringify(oldVal) !== JSON.stringify(newVal)) {
+      changes.push({ path, old: oldVal, new: newVal });
+    }
+  }
+  return changes;
+}
+function countMonitorDiffs(diffs) {
+  let count = 0;
+  if (diffs.markdown) count += diffs.markdown.length;
+  if (diffs.html) count += diffs.html.length;
+  if (diffs.json) count += diffs.json.length;
+  if (diffs.summary) count += diffs.summary.length;
+  if (diffs.links) count += diffs.links.added.length + diffs.links.removed.length;
+  if (diffs.images) count += diffs.images.added.length + diffs.images.removed.length;
+  if (diffs.branding) count += diffs.branding.length;
+  if (diffs.screenshot) count += diffs.screenshot.changed > 0 ? 1 : 0;
+  return count;
+}
 export {
   buildPriorHashMap,
   computeContentHash,
+  computeJsonDiffs,
+  computeSetDiffs,
+  computeTextDiffs,
+  countMonitorDiffs,
   diffContent,
+  flattenJson,
   hashContent,
   normalizeText,
   partitionByChange

package/dist/incremental-diff.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/incremental-diff.ts"],
-  "sourcesContent": ["/**\n * Incremental crawl diff \u2014 compute stable content hashes and skip unchanged screens.\n *\n * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).\n * On the next crawl of the same project, pages with the same fingerprint are\n * marked SKIPPED instead of re-processed, drastically reducing crawl time and\n * LLM token spend for sites with mostly-static content.\n *\n * Design: \"same content hash\" means the meaningful text hasn't changed.\n * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.\n */\n\nexport interface ContentHash {\n  url: string;\n  hash: string;\n  textLength: number;\n  computedAt: string; // ISO timestamp\n}\n\nexport interface DiffResult {\n  url: string;\n  changed: boolean;\n  previousHash?: string;\n  currentHash: string;\n  reason: 'new' | 'changed' | 'unchanged';\n}\n\n/**\n * Compute a stable 64-bit-ish hash from text content.\n * djb2 variant \u2014 fast, no deps, collision-resistant enough for content fingerprinting.\n */\nexport function hashContent(text: string): string {\n  let h1 = 5381;\n  let h2 = 52711;\n  for (let i = 0; i < text.length; i++) {\n    const c = text.charCodeAt(i);\n    h1 = (Math.imul(h1, 33) ^ c) >>> 0;\n    h2 = (Math.imul(h2, 31) ^ c) >>> 0;\n  }\n  return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');\n}\n\n/**\n * Normalize text before hashing \u2014 strips whitespace-only lines, trims each line,\n * collapses runs of blank lines. Makes hash stable across minor formatting changes.\n */\nexport function normalizeText(text: string): string {\n  return text\n    .split('\\n')\n    .map(l => l.trim())\n    .filter(l => l.length > 0)\n    .join('\\n');\n}\n\nexport function computeContentHash(url: string, markdownText: string): ContentHash {\n  const normalized = normalizeText(markdownText);\n  return {\n    url,\n    hash: hashContent(normalized),\n    textLength: normalized.length,\n    computedAt: new Date().toISOString(),\n  };\n}\n\n/**\n * Compare a page's current content against prior hashes.\n * Returns { changed: false } for unchanged pages \u2014 caller should skip re-processing.\n */\nexport function diffContent(\n  url: string,\n  currentMarkdown: string,\n  priorHashes: Map<string, string>, // url \u2192 hash\n): DiffResult {\n  const current = computeContentHash(url, currentMarkdown);\n  const prior = priorHashes.get(url);\n\n  if (!prior) {\n    return { url, changed: true, currentHash: current.hash, reason: 'new' };\n  }\n  if (prior === current.hash) {\n    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };\n  }\n  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };\n}\n\n/**\n * Build a hash map from prior screen records (from DB query result).\n * screenRecords: Array<{ url: string; elementHash?: string | null }>\n */\nexport function buildPriorHashMap(screenRecords: Array<{ url: string; elementHash?: string | null }>): Map<string, string> {\n  const map = new Map<string, string>();\n  for (const s of screenRecords) {\n    if (s.url && s.elementHash) map.set(s.url, s.elementHash);\n  }\n  return map;\n}\n\n/**\n * Filter a batch of discovered URLs against prior hashes.\n * Returns { toProcess, toSkip } \u2014 caller enqueues toProcess, logs toSkip count.\n *\n * Note: filtering at URL level only works if the prior crawl already visited the URL.\n * For new URLs, they're always in toProcess.\n */\nexport function partitionByChange(\n  urls: string[],\n  priorHashes: Map<string, string>,\n): { toProcess: string[]; toSkip: string[] } {\n  const toProcess: string[] = [];\n  const toSkip: string[] = [];\n  for (const url of urls) {\n    if (priorHashes.has(url)) {\n      // Can't know if changed without fetching \u2014 still needs fetch, but hash after fetch\n      // will determine skip. Include all \u2014 hash check happens post-fetch.\n      toProcess.push(url);\n    } else {\n      toProcess.push(url);\n    }\n  }\n  return { toProcess, toSkip };\n}\n"],
-  "mappings": "AA+BO,SAAS,YAAY,MAAsB;AAChD,MAAI,KAAK;AACT,MAAI,KAAK;AACT,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,IAAI,KAAK,WAAW,CAAC;AAC3B,UAAM,KAAK,KAAK,IAAI,EAAE,IAAI,OAAO;AACjC,UAAM,KAAK,KAAK,IAAI,EAAE,IAAI,OAAO;AAAA,EACnC;AACA,UAAQ,OAAO,GAAG,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,KAAK,OAAO,GAAG,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAC3F;AAMO,SAAS,cAAc,MAAsB;AAClD,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,OAAK,EAAE,KAAK,CAAC,EACjB,OAAO,OAAK,EAAE,SAAS,CAAC,EACxB,KAAK,IAAI;AACd;AAEO,SAAS,mBAAmB,KAAa,cAAmC;AACjF,QAAM,aAAa,cAAc,YAAY;AAC7C,SAAO;AAAA,IACL;AAAA,IACA,MAAM,YAAY,UAAU;AAAA,IAC5B,YAAY,WAAW;AAAA,IACvB,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,EACrC;AACF;AAMO,SAAS,YACd,KACA,iBACA,aACY;AACZ,QAAM,UAAU,mBAAmB,KAAK,eAAe;AACvD,QAAM,QAAQ,YAAY,IAAI,GAAG;AAEjC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,KAAK,SAAS,MAAM,aAAa,QAAQ,MAAM,QAAQ,MAAM;AAAA,EACxE;AACA,MAAI,UAAU,QAAQ,MAAM;AAC1B,WAAO,EAAE,KAAK,SAAS,OAAO,cAAc,OAAO,aAAa,QAAQ,MAAM,QAAQ,YAAY;AAAA,EACpG;AACA,SAAO,EAAE,KAAK,SAAS,MAAM,cAAc,OAAO,aAAa,QAAQ,MAAM,QAAQ,UAAU;AACjG;AAMO,SAAS,kBAAkB,eAAyF;AACzH,QAAM,MAAM,oBAAI,IAAoB;AACpC,aAAW,KAAK,eAAe;AAC7B,QAAI,EAAE,OAAO,EAAE,YAAa,KAAI,IAAI,EAAE,KAAK,EAAE,WAAW;AAAA,EAC1D;AACA,SAAO;AACT;AASO,SAAS,kBACd,MACA,aAC2C;AAC3C,QAAM,YAAsB,CAAC;AAC7B,QAAM,SAAmB,CAAC;AAC1B,aAAW,OAAO,MAAM;AACtB,QAAI,YAAY,IAAI,GAAG,GAAG;AAGxB,gBAAU,KAAK,GAAG;AAAA,IACpB,OAAO;AACL,gBAAU,KAAK,GAAG;AAAA,IACpB;AAAA,EACF;AACA,SAAO,EAAE,WAAW,OAAO;AAC7B;",
+  "sourcesContent": ["/**\n * Incremental crawl diff \u2014 compute stable content hashes and skip unchanged screens.\n *\n * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).\n * On the next crawl of the same project, pages with the same fingerprint are\n * marked SKIPPED instead of re-processed, drastically reducing crawl time and\n * LLM token spend for sites with mostly-static content.\n *\n * Design: \"same content hash\" means the meaningful text hasn't changed.\n * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.\n */\n\nexport interface ContentHash {\n  url: string;\n  hash: string;\n  textLength: number;\n  computedAt: string; // ISO timestamp\n}\n\nexport interface DiffResult {\n  url: string;\n  changed: boolean;\n  previousHash?: string;\n  currentHash: string;\n  reason: 'new' | 'changed' | 'unchanged';\n}\n\n/**\n * Compute a stable 64-bit-ish hash from text content.\n * djb2 variant \u2014 fast, no deps, collision-resistant enough for content fingerprinting.\n */\nexport function hashContent(text: string): string {\n  let h1 = 5381;\n  let h2 = 52711;\n  for (let i = 0; i < text.length; i++) {\n    const c = text.charCodeAt(i);\n    h1 = (Math.imul(h1, 33) ^ c) >>> 0;\n    h2 = (Math.imul(h2, 31) ^ c) >>> 0;\n  }\n  return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');\n}\n\n/**\n * Normalize text before hashing \u2014 strips whitespace-only lines, trims each line,\n * collapses runs of blank lines. Makes hash stable across minor formatting changes.\n */\nexport function normalizeText(text: string): string {\n  return text\n    .split('\\n')\n    .map(l => l.trim())\n    .filter(l => l.length > 0)\n    .join('\\n');\n}\n\nexport function computeContentHash(url: string, markdownText: string): ContentHash {\n  const normalized = normalizeText(markdownText);\n  return {\n    url,\n    hash: hashContent(normalized),\n    textLength: normalized.length,\n    computedAt: new Date().toISOString(),\n  };\n}\n\n/**\n * Compare a page's current content against prior hashes.\n * Returns { changed: false } for unchanged pages \u2014 caller should skip re-processing.\n */\nexport function diffContent(\n  url: string,\n  currentMarkdown: string,\n  priorHashes: Map<string, string>, // url \u2192 hash\n): DiffResult {\n  const current = computeContentHash(url, currentMarkdown);\n  const prior = priorHashes.get(url);\n\n  if (!prior) {\n    return { url, changed: true, currentHash: current.hash, reason: 'new' };\n  }\n  if (prior === current.hash) {\n    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };\n  }\n  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };\n}\n\n/**\n * Build a hash map from prior screen records (from DB query result).\n * screenRecords: Array<{ url: string; elementHash?: string | null }>\n */\nexport function buildPriorHashMap(screenRecords: Array<{ url: string; elementHash?: string | null }>): Map<string, string> {\n  const map = new Map<string, string>();\n  for (const s of screenRecords) {\n    if (s.url && s.elementHash) map.set(s.url, s.elementHash);\n  }\n  return map;\n}\n\n/**\n * Filter a batch of discovered URLs against prior hashes.\n * Returns { toProcess, toSkip } \u2014 caller enqueues toProcess, logs toSkip count.\n *\n * Note: filtering at URL level only works if the prior crawl already visited the URL.\n * For new URLs, they're always in toProcess.\n */\nexport function partitionByChange(\n  urls: string[],\n  priorHashes: Map<string, string>,\n): { toProcess: string[]; toSkip: string[] } {\n  const toProcess: string[] = [];\n  const toSkip: string[] = [];\n  for (const url of urls) {\n    if (priorHashes.has(url)) {\n      // Can't know if changed without fetching \u2014 still needs fetch, but hash after fetch\n      // will determine skip. Include all \u2014 hash check happens post-fetch.\n      toProcess.push(url);\n    } else {\n      toProcess.push(url);\n    }\n  }\n  return { toProcess, toSkip };\n}\n\n// \u2500\u2500\u2500 Typed monitor diffs (GAP-01) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n/** A single line-level change in text content (markdown, html, summary). */\nexport interface TextChange {\n  type: 'added' | 'removed';\n  line: number;\n  content: string;\n}\n\n/** A change to a specific JSON path between two scraped JSON objects. */\nexport interface JsonChange {\n  path: string;\n  old: unknown;\n  new: unknown;\n}\n\n/** Set-based change for collections like links or images. */\nexport interface SetChange {\n  added: string[];\n  removed: string[];\n}\n\n/** Screenshot pixel-level change. changed is percentage (0-100). */\nexport interface ImageChange {\n  size: number;\n  changed: number;\n  mask?: string; // base64 diff mask\n}\n\n/** All possible diffs for a single monitor tick. */\nexport interface MonitorDiffs {\n  markdown?: TextChange[];\n  html?: TextChange[];\n  json?: JsonChange[];\n  summary?: TextChange[];\n  links?: SetChange;\n  images?: SetChange;\n  branding?: JsonChange[];\n  screenshot?: ImageChange;\n}\n\n/**\n * Compute line-level text diffs between two strings.\n * Returns TextChange[] showing exactly which lines were added or removed.\n * No external dependencies \u2014 pure JS Myers-inspired algorithm.\n */\nexport function computeTextDiffs(previous: string, current: string): TextChange[] {\n  const prevLines = previous.split('\\n');\n  const currLines = current.split('\\n');\n  const changes: TextChange[] = [];\n\n  const prevSet = new Set(prevLines);\n  const currSet = new Set(currLines);\n\n  let lineNum = 0;\n  for (const line of prevLines) {\n    if (!currSet.has(line)) {\n      changes.push({ type: 'removed', line: lineNum, content: line });\n    }\n    lineNum++;\n  }\n  lineNum = 0;\n  for (const line of currLines) {\n    if (!prevSet.has(line)) {\n      changes.push({ type: 'added', line: lineNum, content: line });\n    }\n    lineNum++;\n  }\n\n  return changes;\n}\n\n/**\n * Compute set-based diffs for URL collections (links, images).\n */\nexport function computeSetDiffs(previous: string[], current: string[]): SetChange {\n  const prevSet = new Set(previous);\n  const currSet = new Set(current);\n  return {\n    added: current.filter(u => !prevSet.has(u)),\n    removed: previous.filter(u => !currSet.has(u)),\n  };\n}\n\n/**\n * Flatten a JSON object into dot-notation path/value pairs.\n * Used to compute JsonChange[] between two JSON extraction results.\n */\nexport function flattenJson(obj: unknown, prefix = ''): Record<string, unknown> {\n  if (typeof obj !== 'object' || obj === null) {\n    return { [prefix]: obj };\n  }\n  const result: Record<string, unknown> = {};\n  for (const [k, v] of Object.entries(obj as Record<string, unknown>)) {\n    const path = prefix ? `${prefix}.${k}` : k;\n    if (typeof v === 'object' && v !== null && !Array.isArray(v)) {\n      Object.assign(result, flattenJson(v, path));\n    } else {\n      result[path] = v;\n    }\n  }\n  return result;\n}\n\n/**\n * Compute JSON path-level diffs between two extraction results.\n */\nexport function computeJsonDiffs(previous: unknown, current: unknown): JsonChange[] {\n  const prevFlat = flattenJson(previous);\n  const currFlat = flattenJson(current);\n  const allKeys = new Set([...Object.keys(prevFlat), ...Object.keys(currFlat)]);\n  const changes: JsonChange[] = [];\n  for (const path of allKeys) {\n    const oldVal = prevFlat[path];\n    const newVal = currFlat[path];\n    if (JSON.stringify(oldVal) !== JSON.stringify(newVal)) {\n      changes.push({ path, old: oldVal, new: newVal });\n    }\n  }\n  return changes;\n}\n\n/**\n * Count total number of changes across all diff types.\n */\nexport function countMonitorDiffs(diffs: MonitorDiffs): number {\n  let count = 0;\n  if (diffs.markdown) count += diffs.markdown.length;\n  if (diffs.html) count += diffs.html.length;\n  if (diffs.json) count += diffs.json.length;\n  if (diffs.summary) count += diffs.summary.length;\n  if (diffs.links) count += diffs.links.added.length + diffs.links.removed.length;\n  if (diffs.images) count += diffs.images.added.length + diffs.images.removed.length;\n  if (diffs.branding) count += diffs.branding.length;\n  if (diffs.screenshot) count += diffs.screenshot.changed > 0 ? 1 : 0;\n  return count;\n}\n"],
+  "mappings": "AA+BO,SAAS,YAAY,MAAsB;AAChD,MAAI,KAAK;AACT,MAAI,KAAK;AACT,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,IAAI,KAAK,WAAW,CAAC;AAC3B,UAAM,KAAK,KAAK,IAAI,EAAE,IAAI,OAAO;AACjC,UAAM,KAAK,KAAK,IAAI,EAAE,IAAI,OAAO;AAAA,EACnC;AACA,UAAQ,OAAO,GAAG,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,KAAK,OAAO,GAAG,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAC3F;AAMO,SAAS,cAAc,MAAsB;AAClD,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,OAAK,EAAE,KAAK,CAAC,EACjB,OAAO,OAAK,EAAE,SAAS,CAAC,EACxB,KAAK,IAAI;AACd;AAEO,SAAS,mBAAmB,KAAa,cAAmC;AACjF,QAAM,aAAa,cAAc,YAAY;AAC7C,SAAO;AAAA,IACL;AAAA,IACA,MAAM,YAAY,UAAU;AAAA,IAC5B,YAAY,WAAW;AAAA,IACvB,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,EACrC;AACF;AAMO,SAAS,YACd,KACA,iBACA,aACY;AACZ,QAAM,UAAU,mBAAmB,KAAK,eAAe;AACvD,QAAM,QAAQ,YAAY,IAAI,GAAG;AAEjC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,KAAK,SAAS,MAAM,aAAa,QAAQ,MAAM,QAAQ,MAAM;AAAA,EACxE;AACA,MAAI,UAAU,QAAQ,MAAM;AAC1B,WAAO,EAAE,KAAK,SAAS,OAAO,cAAc,OAAO,aAAa,QAAQ,MAAM,QAAQ,YAAY;AAAA,EACpG;AACA,SAAO,EAAE,KAAK,SAAS,MAAM,cAAc,OAAO,aAAa,QAAQ,MAAM,QAAQ,UAAU;AACjG;AAMO,SAAS,kBAAkB,eAAyF;AACzH,QAAM,MAAM,oBAAI,IAAoB;AACpC,aAAW,KAAK,eAAe;AAC7B,QAAI,EAAE,OAAO,EAAE,YAAa,KAAI,IAAI,EAAE,KAAK,EAAE,WAAW;AAAA,EAC1D;AACA,SAAO;AACT;AASO,SAAS,kBACd,MACA,aAC2C;AAC3C,QAAM,YAAsB,CAAC;AAC7B,QAAM,SAAmB,CAAC;AAC1B,aAAW,OAAO,MAAM;AACtB,QAAI,YAAY,IAAI,GAAG,GAAG;AAGxB,gBAAU,KAAK,GAAG;AAAA,IACpB,OAAO;AACL,gBAAU,KAAK,GAAG;AAAA,IACpB;AAAA,EACF;AACA,SAAO,EAAE,WAAW,OAAO;AAC7B;AAgDO,SAAS,iBAAiB,UAAkB,SAA+B;AAChF,QAAM,YAAY,SAAS,MAAM,IAAI;AACrC,QAAM,YAAY,QAAQ,MAAM,IAAI;AACpC,QAAM,UAAwB,CAAC;AAE/B,QAAM,UAAU,IAAI,IAAI,SAAS;AACjC,QAAM,UAAU,IAAI,IAAI,SAAS;AAEjC,MAAI,UAAU;AACd,aAAW,QAAQ,WAAW;AAC5B,QAAI,CAAC,QAAQ,IAAI,IAAI,GAAG;AACtB,cAAQ,KAAK,EAAE,MAAM,WAAW,MAAM,SAAS,SAAS,KAAK,CAAC;AAAA,IAChE;AACA;AAAA,EACF;AACA,YAAU;AACV,aAAW,QAAQ,WAAW;AAC5B,QAAI,CAAC,QAAQ,IAAI,IAAI,GAAG;AACtB,cAAQ,KAAK,EAAE,MAAM,SAAS,MAAM,SAAS,SAAS,KAAK,CAAC;AAAA,IAC9D;AACA;AAAA,EACF;AAEA,SAAO;AACT;AAKO,SAAS,gBAAgB,UAAoB,SAA8B;AAChF,QAAM,UAAU,IAAI,IAAI,QAAQ;AAChC,QAAM,UAAU,IAAI,IAAI,OAAO;AAC/B,SAAO;AAAA,IACL,OAAO,QAAQ,OAAO,OAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,IAC1C,SAAS,SAAS,OAAO,OAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EAC/C;AACF;AAMO,SAAS,YAAY,KAAc,SAAS,IAA6B;AAC9E,MAAI,OAAO,QAAQ,YAAY,QAAQ,MAAM;AAC3C,WAAO,EAAE,CAAC,MAAM,GAAG,IAAI;AAAA,EACzB;AACA,QAAM,SAAkC,CAAC;AACzC,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAA8B,GAAG;AACnE,UAAM,OAAO,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK;AACzC,QAAI,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC,GAAG;AAC5D,aAAO,OAAO,QAAQ,YAAY,GAAG,IAAI,CAAC;AAAA,IAC5C,OAAO;AACL,aAAO,IAAI,IAAI;AAAA,IACjB;AAAA,EACF;AACA,SAAO;AACT;AAKO,SAAS,iBAAiB,UAAmB,SAAgC;AAClF,QAAM,WAAW,YAAY,QAAQ;AACrC,QAAM,WAAW,YAAY,OAAO;AACpC,QAAM,UAAU,oBAAI,IAAI,CAAC,GAAG,OAAO,KAAK,QAAQ,GAAG,GAAG,OAAO,KAAK,QAAQ,CAAC,CAAC;AAC5E,QAAM,UAAwB,CAAC;AAC/B,aAAW,QAAQ,SAAS;AAC1B,UAAM,SAAS,SAAS,IAAI;AAC5B,UAAM,SAAS,SAAS,IAAI;AAC5B,QAAI,KAAK,UAAU,MAAM,MAAM,KAAK,UAAU,MAAM,GAAG;AACrD,cAAQ,KAAK,EAAE,MAAM,KAAK,QAAQ,KAAK,OAAO,CAAC;AAAA,IACjD;AAAA,EACF;AACA,SAAO;AACT;AAKO,SAAS,kBAAkB,OAA6B;AAC7D,MAAI,QAAQ;AACZ,MAAI,MAAM,SAAU,UAAS,MAAM,SAAS;AAC5C,MAAI,MAAM,KAAM,UAAS,MAAM,KAAK;AACpC,MAAI,MAAM,KAAM,UAAS,MAAM,KAAK;AACpC,MAAI,MAAM,QAAS,UAAS,MAAM,QAAQ;AAC1C,MAAI,MAAM,MAAO,UAAS,MAAM,MAAM,MAAM,SAAS,MAAM,MAAM,QAAQ;AACzE,MAAI,MAAM,OAAQ,UAAS,MAAM,OAAO,MAAM,SAAS,MAAM,OAAO,QAAQ;AAC5E,MAAI,MAAM,SAAU,UAAS,MAAM,SAAS;AAC5C,MAAI,MAAM,WAAY,UAAS,MAAM,WAAW,UAAU,IAAI,IAAI;AAClE,SAAO;AACT;",
   "names": []
 }

package/dist/index.js

@@ -3,7 +3,7 @@ export * from "./visual-diff-verdict.js";
 export * from "./auth.js";
 export * from "./screen-reader.js";
 export * from "./job-queue.js";
-import { startSqsWorker } from "./sqs-queue.js";
+import { startSqsWorker, getDlqStats, redriveFromDlq } from "./sqs-queue.js";
 export * from "./session-capture.js";
 export * from "./session-export.js";
 export * from "./test-executor.js";
@@ -59,6 +59,7 @@ export {
   extractMarkdown,
   flushMetrics,
   formatLocatorsForPrompt,
+  getDlqStats,
   getFrame,
   getInboxEmail,
   getSsoClientCredentialsToken,
@@ -71,6 +72,7 @@ export {
   maybeCreateRedisQueue,
   prepareExtensionProfile,
   probeTlsFingerprint,
+  redriveFromDlq,
   sendInput,
   startLiveView,
   startSpan,

package/dist/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/index.ts"],
-  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\nexport { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';\nexport type { PlaywrightLocator } from './locator-extractor.js';\nexport * from './dropdown-enumerator.js';\nexport * from './crud-interaction.js';\nexport * from './filter-pagination-crawler.js';\nexport * from './multi-context-handler.js';\nexport * from './advanced-ui-probes.js';\nexport * from './state-graph.js';\nexport * from './nested-window-crawler.js';\nexport * from './shadow-dom-crawler.js';\nexport * from './network-action-correlator.js';\nexport * from './crawl-session-manager.js';\nexport * from './accessibility-tree-crawler.js';\nexport * from './performance-metrics.js';\nexport * from './service-worker-interceptor.js';\n"],
-  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;AAE/B,SAAS,4BAA4B,8BAA8B,yBAAyB,8BAA8B;AAE1H,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;",
+  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker, getDlqStats, redriveFromDlq } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\nexport { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';\nexport type { PlaywrightLocator } from './locator-extractor.js';\nexport * from './dropdown-enumerator.js';\nexport * from './crud-interaction.js';\nexport * from './filter-pagination-crawler.js';\nexport * from './multi-context-handler.js';\nexport * from './advanced-ui-probes.js';\nexport * from './state-graph.js';\nexport * from './nested-window-crawler.js';\nexport * from './shadow-dom-crawler.js';\nexport * from './network-action-correlator.js';\nexport * from './crawl-session-manager.js';\nexport * from './accessibility-tree-crawler.js';\nexport * from './performance-metrics.js';\nexport * from './service-worker-interceptor.js';\n"],
+  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,gBAAgB,aAAa,sBAAsB;AAC5D,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;AAE/B,SAAS,4BAA4B,8BAA8B,yBAAyB,8BAA8B;AAE1H,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;",
   "names": []
 }