@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/http-engine.d.ts

@@ -0,0 +1,80 @@
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
+export interface HttpScreen {
+    url: string;
+    title: string | null;
+    markdown: string;
+    html?: string;
+    metadata: {
+        description: string | null;
+        ogTitle: string | null;
+        ogDescription: string | null;
+        ogImage: string | null;
+        canonical: string | null;
+        schemaTypes: string[];
+        lang: string | null;
+    };
+    tables: Array<Record<string, string>[]>;
+    links: string[];
+    images: Array<{
+        src: string;
+        alt: string;
+    }>;
+    statusCode: number;
+    fromCache: boolean;
+    requiresBrowser?: boolean;
+    durationMs: number;
+}
+export interface HttpCrawlOpts {
+    appUrl: string;
+    maxScreens?: number;
+    urlFilter?: string;
+    traversalStrategy?: 'BFS' | 'DFS';
+    outputFormats?: HttpOutputFormat[];
+    cacheDir?: string;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    respectRobotsTxt?: boolean;
+    includeRawHtml?: boolean;
+    timeoutMs?: number;
+    concurrency?: number;
+    onProgress?: (update: {
+        url: string;
+        screensFound: number;
+        total: number;
+        cached: boolean;
+    }) => void;
+}
+export interface HttpCrawlResult {
+    screens: HttpScreen[];
+    stats: {
+        total: number;
+        cached: number;
+        errors: number;
+        rateLimited: number;
+        durationMs: number;
+        cacheHitRate: number;
+    };
+    jsonLines?: string;
+    csv?: string;
+}
+export declare function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult>;

package/dist/http-engine.js

@@ -0,0 +1,318 @@
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+import { parseHTML } from "linkedom";
+import { RobotsCache } from "./robots-txt.js";
+import { ResponseCache } from "./response-cache.js";
+import { extractMetadata, extractTables, extractImages } from "./extractors/advanced.js";
+import { CHROME_CIPHERS, CHROME_CURVES } from "./tls-fingerprint.js";
+import { renderingPredictor } from "./engine-router.js";
+let chromeFetchDispatcher;
+async function getChromeFetchDispatcher() {
+  if (!chromeFetchDispatcher) {
+    try {
+      const impitMod = await import("impit");
+      const ImpitClass = impitMod.Impit ?? impitMod.default;
+      if (ImpitClass) {
+        chromeFetchDispatcher = new ImpitClass({ browser: "chrome" });
+        return chromeFetchDispatcher;
+      }
+    } catch {
+    }
+    const { Agent } = await import("undici");
+    chromeFetchDispatcher = new Agent({
+      connect: {
+        ciphers: CHROME_CIPHERS,
+        ecdhCurve: CHROME_CURVES,
+        minVersion: "TLSv1.2",
+        maxVersion: "TLSv1.3"
+      },
+      keepAliveTimeout: 1e4,
+      keepAliveMaxTimeout: 3e4
+    });
+  }
+  return chromeFetchDispatcher;
+}
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA ? process.env.CRAWL_SKIP_PATH_EXTRA.split(",").map((p) => p.trim()).filter(Boolean) : [];
+const SKIP_PATH = _skipPathExtra.length > 0 ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join("|")})([-/]|$)`, "i") : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+async function getHttpConfig() {
+  const { getSystemConfig } = await import("@taodigital/core");
+  const cfg = await getSystemConfig("crawl_tuning");
+  return {
+    retryLimit: parseInt(String(cfg?.httpRetryLimit ?? "5"), 10),
+    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs ?? "60000"), 10),
+    retry5xx: parseInt(String(cfg?.http5xxRetryLimit ?? "3"), 10),
+    timeoutMs: parseInt(String(cfg?.httpTimeoutMs ?? "15000"), 10),
+    concurrency: parseInt(String(cfg?.httpConcurrency ?? "5"), 10)
+  };
+}
+function canonicalize(url) {
+  try {
+    const u = new URL(url);
+    u.hash = "";
+    for (const p of ["utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term", "fbclid", "gclid", "ref", "referrer"]) {
+      u.searchParams.delete(p);
+    }
+    return u.href.toLowerCase().replace(/\/$/, "");
+  } catch {
+    return url.toLowerCase();
+  }
+}
+function extractLinks(html, baseUrl) {
+  const links = [];
+  const seen = /* @__PURE__ */ new Set();
+  try {
+    const base = new URL(baseUrl);
+    const { document } = parseHTML(html);
+    const anchors = Array.from(document.querySelectorAll("a[href]"));
+    for (const a of anchors) {
+      const href = a.getAttribute("href") ?? "";
+      if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) continue;
+      try {
+        const resolved = new URL(href, base).href;
+        if (new URL(resolved).origin !== base.origin) continue;
+        const c = canonicalize(resolved);
+        if (!seen.has(c)) {
+          seen.add(c);
+          links.push(resolved);
+        }
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return links;
+}
+function extractMarkdown(html, url) {
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll("script, style, noscript"))) tag.remove();
+    const reader = new Readability(document, { charThreshold: 50 });
+    const article = reader.parse();
+    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
+      const md = turndown.turndown(article.content ?? html);
+      return { markdown: md, title: article.title ?? null };
+    }
+  } catch {
+  }
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll("script, style, noscript, nav, footer"))) tag.remove();
+    const bodyHtml = document.body?.innerHTML ?? html;
+    const md = turndown.turndown(bodyHtml);
+    const titleEl = document.querySelector("title");
+    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
+  } catch {
+    return { markdown: "", title: null };
+  }
+}
+async function fetchWithBackoff(url, cache, opts, retryCount = 0, httpCfg) {
+  if (!httpCfg) httpCfg = await getHttpConfig();
+  const t0 = performance.now();
+  if (cache.isFresh(url)) {
+    const cached = cache.get(url);
+    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
+  }
+  const validationHeaders = cache.getValidationHeaders(url);
+  const headers = {
+    "User-Agent": opts.userAgent,
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    // Chrome-like security headers reduce bot-detection fingerprint
+    "sec-ch-ua": `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? "120"}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? "120"}", "Not-A.Brand";v="99"`,
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Windows"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    ...validationHeaders,
+    ...opts.extraHeaders ?? {}
+  };
+  let response;
+  try {
+    const dispatcher = await getChromeFetchDispatcher();
+    response = await fetch(url, {
+      headers,
+      signal: AbortSignal.timeout(opts.timeoutMs),
+      redirect: "follow",
+      dispatcher
+    });
+  } catch (e) {
+    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
+  }
+  const durationMs = performance.now() - t0;
+  if (response.status === 304) {
+    const cached = cache.get(url);
+    if (cached) {
+      cache.markHit(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
+    }
+  }
+  if (response.status === 429 || response.status === 503) {
+    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
+    const retryAfter = parseInt(response.headers.get("retry-after") ?? "0") || 0;
+    const baseDelay = retryAfter > 0 ? retryAfter * 1e3 : Math.min(httpCfg.backoffCapMs, 1e3 * Math.pow(2, retryCount));
+    const jitter = Math.random() * 1e3;
+    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
+    await new Promise((r) => setTimeout(r, baseDelay + jitter));
+    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
+  }
+  if (response.status >= 500) {
+    if (retryCount < httpCfg.retry5xx) {
+      await new Promise((r) => setTimeout(r, 2e3 * (retryCount + 1)));
+      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
+    }
+    if (cache.staleIfError(url)) {
+      const cached = cache.get(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
+    }
+  }
+  const contentType = response.headers.get("content-type") ?? "";
+  const body = await response.text();
+  const etag = response.headers.get("etag") ?? void 0;
+  const lastModified = response.headers.get("last-modified") ?? void 0;
+  const cacheControl = response.headers.get("cache-control") ?? void 0;
+  if (response.ok && contentType.includes("html")) {
+    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
+  }
+  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
+}
+async function httpCrawl(opts) {
+  const startTime = performance.now();
+  const httpCfg = await getHttpConfig();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? "/tmp/zeta-cache";
+  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
+  const userAgent = opts.userAgent ?? "ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)";
+  const cache = new ResponseCache(cacheDir);
+  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
+  const urlFilterRe = opts.urlFilter ? (() => {
+    try {
+      return new RegExp(opts.urlFilter, "i");
+    } catch {
+      return null;
+    }
+  })() : null;
+  const visited = /* @__PURE__ */ new Set();
+  const queue = [opts.appUrl];
+  const queued = /* @__PURE__ */ new Set([canonicalize(opts.appUrl)]);
+  const screens = [];
+  let cachedCount = 0;
+  let errorCount = 0;
+  let rateLimitedCount = 0;
+  const batchFetch = async (urls) => {
+    await Promise.all(urls.map(async (url) => {
+      const canonUrl = canonicalize(url);
+      if (visited.has(canonUrl)) return;
+      visited.add(canonUrl);
+      if (robots && !await robots.isAllowed(url).catch(() => true)) {
+        console.log(`[http-engine] robots.txt blocked: ${url}`);
+        return;
+      }
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) return;
+      } catch {
+      }
+      if (urlFilterRe) {
+        try {
+          const { pathname, search } = new URL(url);
+          if (!urlFilterRe.test(pathname + search)) {
+            console.log(`[http-engine] filtered out: ${url}`);
+            return;
+          }
+        } catch {
+        }
+      }
+      let fetchResult;
+      try {
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
+        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
+      } catch (err) {
+        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
+        errorCount++;
+        return;
+      }
+      if (!fetchResult.contentType.includes("html")) return;
+      if (fetchResult.statusCode >= 400) return;
+      if (fetchResult.fromCache) cachedCount++;
+      const { markdown, title } = extractMarkdown(fetchResult.body, url);
+      const fullMeta = extractMetadata(fetchResult.body, url);
+      const links = extractLinks(fetchResult.body, url);
+      const tables = extractTables(fetchResult.body);
+      const images = extractImages(fetchResult.body, url);
+      const screen = {
+        url,
+        title: title ?? fullMeta.title,
+        markdown,
+        html: opts.includeRawHtml ? fetchResult.body : void 0,
+        metadata: {
+          description: fullMeta.description,
+          ogTitle: fullMeta.ogTitle,
+          ogDescription: fullMeta.ogDescription,
+          ogImage: fullMeta.ogImage,
+          canonical: fullMeta.canonical,
+          schemaTypes: fullMeta.schemaTypes,
+          lang: fullMeta.lang
+        },
+        tables,
+        links,
+        images: images.map((i) => ({ src: i.src, alt: i.alt })),
+        statusCode: fetchResult.statusCode,
+        fromCache: fetchResult.fromCache,
+        requiresBrowser: renderingPredictor.getCached(url) === "browser",
+        durationMs: fetchResult.durationMs
+      };
+      screens.push(screen);
+      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
+      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
+      for (const link of links) {
+        const c = canonicalize(link);
+        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
+          queued.add(c);
+          if (opts.traversalStrategy === "DFS") queue.unshift(link);
+          else queue.push(link);
+        }
+      }
+    }));
+  };
+  const concurrency = opts.concurrency ?? httpCfg.concurrency;
+  while (queue.length > 0 && screens.length < maxScreens) {
+    const batch = queue.splice(0, concurrency);
+    await batchFetch(batch);
+  }
+  let jsonLines;
+  let csv;
+  const formats = opts.outputFormats ?? [];
+  if (formats.includes("jsonlines")) {
+    jsonLines = screens.map((s) => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join("\n");
+  }
+  if (formats.includes("csv")) {
+    const header = "url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs";
+    const rows = screens.map(
+      (s) => [s.url, s.title ?? "", s.metadata.description ?? "", s.metadata.ogTitle ?? "", s.metadata.schemaTypes.join("|"), s.statusCode, s.fromCache, Math.round(s.durationMs)].map((v) => `"${String(v).replace(/"/g, '""')}"`).join(",")
+    );
+    csv = [header, ...rows].join("\n");
+  }
+  const cacheStats = cache.stats();
+  return {
+    screens,
+    stats: {
+      total: screens.length,
+      cached: cachedCount,
+      errors: errorCount,
+      rateLimited: rateLimitedCount,
+      durationMs: performance.now() - startTime,
+      cacheHitRate: cacheStats.hitRate
+    },
+    jsonLines,
+    csv
+  };
+}
+export {
+  httpCrawl
+};
+//# sourceMappingURL=http-engine.js.map

package/dist/http-engine.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/http-engine.ts"],
+  "sourcesContent": ["/**\n * HTTP-only crawl engine \u2014 10-50x faster than browser for SSR/static sites.\n *\n * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.\n * No browser overhead. Falls back to browser engine on JS-heavy sites.\n *\n * Features:\n *   - HTTP/2 native (Node 20 built-in fetch)\n *   - ETag / Last-Modified response caching\n *   - 429 / 503 exponential backoff with jitter\n *   - BFS / DFS traversal\n *   - URL regex filtering\n *   - Robots.txt compliance\n *   - Link discovery (href, action, src)\n *   - Markdown extraction (Readability + Turndown)\n *   - OG / schema.org / meta extraction\n *   - Table \u2192 JSON extraction\n *   - JSON Lines, CSV, raw HTML output formats\n *   - Streaming progress via onProgress callback\n */\n\nimport { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { parseHTML } from 'linkedom';\nimport { RobotsCache } from './robots-txt.js';\nimport { ResponseCache } from './response-cache.js';\nimport { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';\nimport { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';\nimport { renderingPredictor } from './engine-router.js';\n\n// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback\nlet chromeFetchDispatcher: any;\nasync function getChromeFetchDispatcher() {\n  if (!chromeFetchDispatcher) {\n    try {\n      // impit provides native Chrome BoringSSL TLS fingerprint \u2014 true JA3/JA4 match\n      // @ts-ignore \u2014 impit is an optional dependency\n      const impitMod = await import('impit');\n      const ImpitClass = impitMod.Impit ?? impitMod.default;\n      if (ImpitClass) {\n        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });\n        return chromeFetchDispatcher;\n      }\n    } catch { /* impit not installed \u2014 fall through to undici */ }\n    const { Agent } = await import('undici');\n    chromeFetchDispatcher = new Agent({\n      connect: {\n        ciphers: CHROME_CIPHERS,\n        ecdhCurve: CHROME_CURVES,\n        minVersion: 'TLSv1.2' as any,\n        maxVersion: 'TLSv1.3' as any,\n      },\n      keepAliveTimeout: 10_000,\n      keepAliveMaxTimeout: 30_000,\n    });\n  }\n  return chromeFetchDispatcher;\n}\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\n\n// Pages unlikely to have testable UI \u2014 skip from HTTP crawl queue\nconst _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA\n  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)\n  : [];\nconst SKIP_PATH = _skipPathExtra.length > 0\n  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')\n  : /\\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;\n\nasync function getHttpConfig() {\n  const { getSystemConfig } = await import('@detiq/core');\n  const cfg = await getSystemConfig('crawl_tuning');\n  return {\n    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),\n    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),\n    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),\n    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),\n    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),\n  };\n}\n\nexport type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';\n\nexport interface HttpScreen {\n  url: string;\n  title: string | null;\n  markdown: string;\n  html?: string;\n  metadata: {\n    description: string | null;\n    ogTitle: string | null;\n    ogDescription: string | null;\n    ogImage: string | null;\n    canonical: string | null;\n    schemaTypes: string[];\n    lang: string | null;\n  };\n  tables: Array<Record<string, string>[]>;\n  links: string[];\n  images: Array<{ src: string; alt: string }>;\n  statusCode: number;\n  fromCache: boolean;\n  requiresBrowser?: boolean;\n  durationMs: number;\n}\n\nexport interface HttpCrawlOpts {\n  appUrl: string;\n  maxScreens?: number;            // default 100\n  urlFilter?: string;             // regex pattern \u2014 only crawl matching URLs\n  traversalStrategy?: 'BFS' | 'DFS'; // default BFS\n  outputFormats?: HttpOutputFormat[];\n  cacheDir?: string;              // default '/tmp/zeta-cache'\n  userAgent?: string;\n  headers?: Record<string, string>;\n  respectRobotsTxt?: boolean;     // default true\n  includeRawHtml?: boolean;\n  timeoutMs?: number;             // per-request timeout, default 15000\n  concurrency?: number;           // default 5 parallel fetches\n  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;\n}\n\nexport interface HttpCrawlResult {\n  screens: HttpScreen[];\n  stats: {\n    total: number;\n    cached: number;\n    errors: number;\n    rateLimited: number;\n    durationMs: number;\n    cacheHitRate: number;\n  };\n  jsonLines?: string;\n  csv?: string;\n}\n\nfunction canonicalize(url: string): string {\n  try {\n    const u = new URL(url);\n    u.hash = '';\n    // Strip tracking params\n    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {\n      u.searchParams.delete(p);\n    }\n    return u.href.toLowerCase().replace(/\\/$/, '');\n  } catch {\n    return url.toLowerCase();\n  }\n}\n\nfunction extractLinks(html: string, baseUrl: string): string[] {\n  const links: string[] = [];\n  const seen = new Set<string>();\n  try {\n    const base = new URL(baseUrl);\n    const { document } = parseHTML(html);\n    const anchors = Array.from(document.querySelectorAll('a[href]'));\n    for (const a of anchors) {\n      const href = (a as any).getAttribute('href') ?? '';\n      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;\n      try {\n        const resolved = new URL(href, base).href;\n        // Same origin only\n        if (new URL(resolved).origin !== base.origin) continue;\n        const c = canonicalize(resolved);\n        if (!seen.has(c)) { seen.add(c); links.push(resolved); }\n      } catch { /* skip */ }\n    }\n  } catch { /* non-fatal */ }\n  return links;\n}\n\nfunction extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {\n  try {\n    const { document } = parseHTML(html);\n    // Remove script/style tags before Readability\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();\n    const reader = new Readability(document as any, { charThreshold: 50 });\n    const article = reader.parse();\n    if (article && article.textContent && article.textContent.split(/\\s+/).length >= 30) {\n      const md = turndown.turndown(article.content ?? html);\n      return { markdown: md, title: article.title ?? null };\n    }\n  } catch { /* fall through */ }\n  // Full-page fallback\n  try {\n    const { document } = parseHTML(html);\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();\n    const bodyHtml = (document as any).body?.innerHTML ?? html;\n    const md = turndown.turndown(bodyHtml);\n    const titleEl = (document as any).querySelector('title');\n    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };\n  } catch {\n    return { markdown: '', title: null };\n  }\n}\n\ninterface FetchResult {\n  body: string;\n  statusCode: number;\n  contentType: string;\n  fromCache: boolean;\n  durationMs: number;\n  etag?: string;\n  lastModified?: string;\n}\n\nasync function fetchWithBackoff(\n  url: string,\n  cache: ResponseCache,\n  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },\n  retryCount = 0,\n  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>\n): Promise<FetchResult> {\n  if (!httpCfg) httpCfg = await getHttpConfig();\n  const t0 = performance.now();\n\n  if (cache.isFresh(url)) {\n    const cached = cache.get(url)!;\n    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };\n  }\n\n  const validationHeaders = cache.getValidationHeaders(url);\n\n  const headers: Record<string, string> = {\n    'User-Agent': opts.userAgent,\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n    'Accept-Language': 'en-US,en;q=0.9',\n    'Accept-Encoding': 'gzip, deflate, br, zstd',\n    // Chrome-like security headers reduce bot-detection fingerprint\n    'sec-ch-ua': `\"Google Chrome\";v=\"${process.env.CRAWLER_CHROME_VERSION ?? '120'}\", \"Chromium\";v=\"${process.env.CRAWLER_CHROME_VERSION ?? '120'}\", \"Not-A.Brand\";v=\"99\"`,\n    'sec-ch-ua-mobile': '?0',\n    'sec-ch-ua-platform': '\"Windows\"',\n    'Sec-Fetch-Dest': 'document',\n    'Sec-Fetch-Mode': 'navigate',\n    'Sec-Fetch-Site': 'none',\n    'Sec-Fetch-User': '?1',\n    'Upgrade-Insecure-Requests': '1',\n    ...validationHeaders,\n    ...(opts.extraHeaders ?? {}),\n  };\n\n  let response: Response;\n  try {\n    const dispatcher = await getChromeFetchDispatcher();\n    response = await (fetch as any)(url, {\n      headers,\n      signal: AbortSignal.timeout(opts.timeoutMs),\n      redirect: 'follow',\n      dispatcher,\n    });\n  } catch (e: any) {\n    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);\n  }\n\n  const durationMs = performance.now() - t0;\n\n  // 304 Not Modified \u2014 use cached body\n  if (response.status === 304) {\n    const cached = cache.get(url);\n    if (cached) {\n      cache.markHit(url);\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };\n    }\n  }\n\n  // 429 / 503 \u2014 exponential backoff with jitter\n  if (response.status === 429 || response.status === 503) {\n    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);\n    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;\n    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));\n    const jitter = Math.random() * 1000;\n    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);\n    await new Promise(r => setTimeout(r, baseDelay + jitter));\n    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);\n  }\n\n  // 5xx \u2014 retry up to 3 times\n  if (response.status >= 500) {\n    if (retryCount < httpCfg.retry5xx) {\n      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));\n      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);\n    }\n    if (cache.staleIfError(url)) {\n      const cached = cache.get(url)!;\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };\n    }\n  }\n\n  const contentType = response.headers.get('content-type') ?? '';\n  const body = await response.text();\n  const etag = response.headers.get('etag') ?? undefined;\n  const lastModified = response.headers.get('last-modified') ?? undefined;\n  const cacheControl = response.headers.get('cache-control') ?? undefined;\n\n  // Cache successful HTML responses\n  if (response.ok && contentType.includes('html')) {\n    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });\n  }\n\n  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };\n}\n\nexport async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {\n  const startTime = performance.now();\n  const httpCfg = await getHttpConfig();\n  const maxScreens = opts.maxScreens ?? 100;\n  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';\n  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;\n  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';\n  const cache = new ResponseCache(cacheDir);\n  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;\n  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;\n\n  const visited = new Set<string>();\n  const queue: string[] = [opts.appUrl];\n  const queued = new Set<string>([canonicalize(opts.appUrl)]);\n  const screens: HttpScreen[] = [];\n  let cachedCount = 0;\n  let errorCount = 0;\n  let rateLimitedCount = 0;\n\n  const batchFetch = async (urls: string[]): Promise<void> => {\n    await Promise.all(urls.map(async (url) => {\n      const canonUrl = canonicalize(url);\n      if (visited.has(canonUrl)) return;\n      visited.add(canonUrl);\n\n      // Robots.txt check\n      if (robots && !(await robots.isAllowed(url).catch(() => true))) {\n        console.log(`[http-engine] robots.txt blocked: ${url}`);\n        return;\n      }\n\n      // Skip content/legal pages\n      try {\n        const { pathname } = new URL(url);\n        if (SKIP_PATH.test(pathname)) return;\n      } catch { /* skip */ }\n\n      // URL filter\n      if (urlFilterRe) {\n        try {\n          const { pathname, search } = new URL(url);\n          if (!urlFilterRe.test(pathname + search)) {\n            console.log(`[http-engine] filtered out: ${url}`);\n            return;\n          }\n        } catch { /* skip */ }\n      }\n\n      let fetchResult: FetchResult;\n      try {\n        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);\n        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;\n      } catch (err: any) {\n        console.error(`[http-engine] error fetching ${url}: ${err.message}`);\n        errorCount++;\n        return;\n      }\n\n      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML\n      if (fetchResult.statusCode >= 400) return;\n      if (fetchResult.fromCache) cachedCount++;\n\n      const { markdown, title } = extractMarkdown(fetchResult.body, url);\n      const fullMeta = extractMetadata(fetchResult.body, url);\n      const links = extractLinks(fetchResult.body, url);\n      const tables = extractTables(fetchResult.body);\n      const images = extractImages(fetchResult.body, url);\n\n      const screen: HttpScreen = {\n        url,\n        title: title ?? fullMeta.title,\n        markdown,\n        html: opts.includeRawHtml ? fetchResult.body : undefined,\n        metadata: {\n          description: fullMeta.description,\n          ogTitle: fullMeta.ogTitle,\n          ogDescription: fullMeta.ogDescription,\n          ogImage: fullMeta.ogImage,\n          canonical: fullMeta.canonical,\n          schemaTypes: fullMeta.schemaTypes,\n          lang: fullMeta.lang,\n        },\n        tables,\n        links,\n        images: images.map(i => ({ src: i.src, alt: i.alt })),\n        statusCode: fetchResult.statusCode,\n        fromCache: fetchResult.fromCache,\n        requiresBrowser: renderingPredictor.getCached(url) === 'browser',\n        durationMs: fetchResult.durationMs,\n      };\n      screens.push(screen);\n\n      // Record rendering prediction: SPA signals \u2192 browser needed; content-rich \u2192 http engine works\n      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);\n\n      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });\n\n      // Enqueue discovered links\n      for (const link of links) {\n        const c = canonicalize(link);\n        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {\n          queued.add(c);\n          if (opts.traversalStrategy === 'DFS') queue.unshift(link);\n          else queue.push(link);\n        }\n      }\n    }));\n  };\n\n  const concurrency = opts.concurrency ?? httpCfg.concurrency;\n\n  while (queue.length > 0 && screens.length < maxScreens) {\n    const batch = queue.splice(0, concurrency);\n    await batchFetch(batch);\n  }\n\n  // Output formats\n  let jsonLines: string | undefined;\n  let csv: string | undefined;\n\n  const formats = opts.outputFormats ?? [];\n  if (formats.includes('jsonlines')) {\n    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\\n');\n  }\n  if (formats.includes('csv')) {\n    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';\n    const rows = screens.map(s =>\n      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `\"${String(v).replace(/\"/g, '\"\"')}\"`).join(',')\n    );\n    csv = [header, ...rows].join('\\n');\n  }\n\n  const cacheStats = cache.stats();\n  return {\n    screens,\n    stats: {\n      total: screens.length,\n      cached: cachedCount,\n      errors: errorCount,\n      rateLimited: rateLimitedCount,\n      durationMs: performance.now() - startTime,\n      cacheHitRate: cacheStats.hitRate,\n    },\n    jsonLines,\n    csv,\n  };\n}\n"],
+  "mappings": "AAwBA,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB,eAAe,qBAAqB;AAC9D,SAAS,gBAAgB,qBAAqB;AAC9C,SAAS,0BAA0B;AAGnC,IAAI;AACJ,eAAe,2BAA2B;AACxC,MAAI,CAAC,uBAAuB;AAC1B,QAAI;AAGF,YAAM,WAAW,MAAM,OAAO,OAAO;AACrC,YAAM,aAAa,SAAS,SAAS,SAAS;AAC9C,UAAI,YAAY;AACd,gCAAwB,IAAI,WAAW,EAAE,SAAS,SAAS,CAAC;AAC5D,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAqD;AAC7D,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,4BAAwB,IAAI,MAAM;AAAA,MAChC,SAAS;AAAA,QACP,SAAS;AAAA,QACT,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,MACd;AAAA,MACA,kBAAkB;AAAA,MAClB,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AAGtF,MAAM,iBAAiB,QAAQ,IAAI,wBAC/B,QAAQ,IAAI,sBAAsB,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO,IAChF,CAAC;AACL,MAAM,YAAY,eAAe,SAAS,IACtC,IAAI,OAAO,8JAA8J,eAAe,KAAK,GAAG,CAAC,aAAa,GAAG,IACjN;AAEJ,eAAe,gBAAgB;AAC7B,QAAM,EAAE,gBAAgB,IAAI,MAAM,OAAO,aAAa;AACtD,QAAM,MAAM,MAAM,gBAAgB,cAAc;AAChD,SAAO;AAAA,IACL,YAAc,SAAS,OAAO,KAAK,kBAAqB,GAAG,GAAO,EAAE;AAAA,IACpE,cAAc,SAAS,OAAO,KAAK,oBAAqB,OAAO,GAAG,EAAE;AAAA,IACpE,UAAc,SAAS,OAAO,KAAK,qBAAqB,GAAG,GAAO,EAAE;AAAA,IACpE,WAAc,SAAS,OAAO,KAAK,iBAAqB,OAAO,GAAG,EAAE;AAAA,IACpE,aAAc,SAAS,OAAO,KAAK,mBAAqB,GAAG,GAAO,EAAE;AAAA,EACtE;AACF;AAyDA,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AAET,eAAW,KAAK,CAAC,cAAc,cAAc,gBAAgB,eAAe,YAAY,UAAU,SAAS,OAAO,UAAU,GAAG;AAC7H,QAAE,aAAa,OAAO,CAAC;AAAA,IACzB;AACA,WAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,OAAO,EAAE;AAAA,EAC/C,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;AAEA,SAAS,aAAa,MAAc,SAA2B;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC;AAC/D,eAAW,KAAK,SAAS;AACvB,YAAM,OAAQ,EAAU,aAAa,MAAM,KAAK;AAChD,UAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,EAAG;AAC9H,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,MAAM,IAAI,EAAE;AAErC,YAAI,IAAI,IAAI,QAAQ,EAAE,WAAW,KAAK,OAAQ;AAC9C,cAAM,IAAI,aAAa,QAAQ;AAC/B,YAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAAE,eAAK,IAAI,CAAC;AAAG,gBAAM,KAAK,QAAQ;AAAA,QAAG;AAAA,MACzD,QAAQ;AAAA,MAAa;AAAA,IACvB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,SAAS,gBAAgB,MAAc,KAAyD;AAC9F,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,yBAAyB,CAAC,EAAG,CAAC,IAAY,OAAO;AACxG,UAAM,SAAS,IAAI,YAAY,UAAiB,EAAE,eAAe,GAAG,CAAC;AACrE,UAAM,UAAU,OAAO,MAAM;AAC7B,QAAI,WAAW,QAAQ,eAAe,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,IAAI;AACnF,YAAM,KAAK,SAAS,SAAS,QAAQ,WAAW,IAAI;AACpD,aAAO,EAAE,UAAU,IAAI,OAAO,QAAQ,SAAS,KAAK;AAAA,IACtD;AAAA,EACF,QAAQ;AAAA,EAAqB;AAE7B,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,sCAAsC,CAAC,EAAG,CAAC,IAAY,OAAO;AACrH,UAAM,WAAY,SAAiB,MAAM,aAAa;AACtD,UAAM,KAAK,SAAS,SAAS,QAAQ;AACrC,UAAM,UAAW,SAAiB,cAAc,OAAO;AACvD,WAAO,EAAE,UAAU,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,KAAK;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,UAAU,IAAI,OAAO,KAAK;AAAA,EACrC;AACF;AAYA,eAAe,iBACb,KACA,OACA,MACA,aAAa,GACb,SACsB;AACtB,MAAI,CAAC,QAAS,WAAU,MAAM,cAAc;AAC5C,QAAM,KAAK,YAAY,IAAI;AAE3B,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,WAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,EAAE;AAAA,EAC/G;AAEA,QAAM,oBAAoB,MAAM,qBAAqB,GAAG;AAExD,QAAM,UAAkC;AAAA,IACtC,cAAc,KAAK;AAAA,IACnB,UAAU;AAAA,IACV,mBAAmB;AAAA,IACnB,mBAAmB;AAAA;AAAA,IAEnB,aAAa,sBAAsB,QAAQ,IAAI,0BAA0B,KAAK,oBAAoB,QAAQ,IAAI,0BAA0B,KAAK;AAAA,IAC7I,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,6BAA6B;AAAA,IAC7B,GAAG;AAAA,IACH,GAAI,KAAK,gBAAgB,CAAC;AAAA,EAC5B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,MAAM,yBAAyB;AAClD,eAAW,MAAO,MAAc,KAAK;AAAA,MACnC;AAAA,MACA,QAAQ,YAAY,QAAQ,KAAK,SAAS;AAAA,MAC1C,UAAU;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH,SAAS,GAAQ;AACf,UAAM,IAAI,MAAM,kCAAkC,GAAG,KAAK,EAAE,OAAO,EAAE;AAAA,EACvE;AAEA,QAAM,aAAa,YAAY,IAAI,IAAI;AAGvC,MAAI,SAAS,WAAW,KAAK;AAC3B,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV,YAAM,QAAQ,GAAG;AACjB,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,WAAW;AAAA,IAC5G;AAAA,EACF;AAGA,MAAI,SAAS,WAAW,OAAO,SAAS,WAAW,KAAK;AACtD,QAAI,cAAc,QAAQ,WAAY,OAAM,IAAI,MAAM,oCAAoC,QAAQ,UAAU,aAAa,GAAG,EAAE;AAC9H,UAAM,aAAa,SAAS,SAAS,QAAQ,IAAI,aAAa,KAAK,GAAG,KAAK;AAC3E,UAAM,YAAY,aAAa,IAAI,aAAa,MAAO,KAAK,IAAI,QAAQ,cAAc,MAAQ,KAAK,IAAI,GAAG,UAAU,CAAC;AACrH,UAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAQ,IAAI,iBAAiB,SAAS,MAAM,OAAO,GAAG,uBAAkB,KAAK,MAAM,YAAY,MAAM,CAAC,aAAa,aAAa,CAAC,IAAI,QAAQ,UAAU,GAAG;AAC1J,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,YAAY,MAAM,CAAC;AACxD,WAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,GAAG,OAAO;AAAA,EACnE;AAGA,MAAI,SAAS,UAAU,KAAK;AAC1B,QAAI,aAAa,QAAQ,UAAU;AACjC,YAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,OAAS,aAAa,EAAE,CAAC;AAC9D,aAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,GAAG,OAAO;AAAA,IACnE;AACA,QAAI,MAAM,aAAa,GAAG,GAAG;AAC3B,YAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,YAAY,IAAI,IAAI,GAAG;AAAA,IACpI;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAM,OAAO,MAAM,SAAS,KAAK;AACjC,QAAM,OAAO,SAAS,QAAQ,IAAI,MAAM,KAAK;AAC7C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAC9D,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAG9D,MAAI,SAAS,MAAM,YAAY,SAAS,MAAM,GAAG;AAC/C,UAAM,IAAI,KAAK,EAAE,KAAK,MAAM,cAAc,cAAc,MAAM,aAAa,YAAY,SAAS,OAAO,CAAC;AAAA,EAC1G;AAEA,SAAO,EAAE,MAAM,YAAY,SAAS,QAAQ,aAAa,WAAW,OAAO,YAAY,MAAM,aAAa;AAC5G;AAEA,eAAsB,UAAU,MAA+C;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,UAAU,MAAM,cAAc;AACpC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,YAAY,KAAK,aAAa,QAAQ;AAC5C,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,QAAQ,IAAI,cAAc,QAAQ;AACxC,QAAM,SAAS,KAAK,qBAAqB,QAAQ,IAAI,YAAY,IAAI;AACrE,QAAM,cAAc,KAAK,aAAa,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,OAAO,KAAK,WAAY,GAAG;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAM;AAAA,EAAE,GAAG,IAAI;AAE7H,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,QAAkB,CAAC,KAAK,MAAM;AACpC,QAAM,SAAS,oBAAI,IAAY,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;AAC1D,QAAM,UAAwB,CAAC;AAC/B,MAAI,cAAc;AAClB,MAAI,aAAa;AACjB,MAAI,mBAAmB;AAEvB,QAAM,aAAa,OAAO,SAAkC;AAC1D,UAAM,QAAQ,IAAI,KAAK,IAAI,OAAO,QAAQ;AACxC,YAAM,WAAW,aAAa,GAAG;AACjC,UAAI,QAAQ,IAAI,QAAQ,EAAG;AAC3B,cAAQ,IAAI,QAAQ;AAGpB,UAAI,UAAU,CAAE,MAAM,OAAO,UAAU,GAAG,EAAE,MAAM,MAAM,IAAI,GAAI;AAC9D,gBAAQ,IAAI,qCAAqC,GAAG,EAAE;AACtD;AAAA,MACF;AAGA,UAAI;AACF,cAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,YAAI,UAAU,KAAK,QAAQ,EAAG;AAAA,MAChC,QAAQ;AAAA,MAAa;AAGrB,UAAI,aAAa;AACf,YAAI;AACF,gBAAM,EAAE,UAAU,OAAO,IAAI,IAAI,IAAI,GAAG;AACxC,cAAI,CAAC,YAAY,KAAK,WAAW,MAAM,GAAG;AACxC,oBAAQ,IAAI,+BAA+B,GAAG,EAAE;AAChD;AAAA,UACF;AAAA,QACF,QAAQ;AAAA,QAAa;AAAA,MACvB;AAEA,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,iBAAiB,KAAK,OAAO,EAAE,WAAW,WAAW,cAAc,KAAK,QAAQ,GAAG,GAAG,OAAO;AACjH,YAAI,YAAY,eAAe,OAAO,YAAY,eAAe,IAAK;AAAA,MACxE,SAAS,KAAU;AACjB,gBAAQ,MAAM,gCAAgC,GAAG,KAAK,IAAI,OAAO,EAAE;AACnE;AACA;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,YAAY,SAAS,MAAM,EAAG;AAC/C,UAAI,YAAY,cAAc,IAAK;AACnC,UAAI,YAAY,UAAW;AAE3B,YAAM,EAAE,UAAU,MAAM,IAAI,gBAAgB,YAAY,MAAM,GAAG;AACjE,YAAM,WAAW,gBAAgB,YAAY,MAAM,GAAG;AACtD,YAAM,QAAQ,aAAa,YAAY,MAAM,GAAG;AAChD,YAAM,SAAS,cAAc,YAAY,IAAI;AAC7C,YAAM,SAAS,cAAc,YAAY,MAAM,GAAG;AAElD,YAAM,SAAqB;AAAA,QACzB;AAAA,QACA,OAAO,SAAS,SAAS;AAAA,QACzB;AAAA,QACA,MAAM,KAAK,iBAAiB,YAAY,OAAO;AAAA,QAC/C,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,SAAS,SAAS;AAAA,UAClB,eAAe,SAAS;AAAA,UACxB,SAAS,SAAS;AAAA,UAClB,WAAW,SAAS;AAAA,UACpB,aAAa,SAAS;AAAA,UACtB,MAAM,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE;AAAA,QACpD,YAAY,YAAY;AAAA,QACxB,WAAW,YAAY;AAAA,QACvB,iBAAiB,mBAAmB,UAAU,GAAG,MAAM;AAAA,QACvD,YAAY,YAAY;AAAA,MAC1B;AACA,cAAQ,KAAK,MAAM;AAGnB,yBAAmB,QAAQ,KAAK,YAAY,MAAM,YAAY,UAAU;AAExE,WAAK,aAAa,EAAE,KAAK,cAAc,QAAQ,QAAQ,OAAO,YAAY,QAAQ,YAAY,UAAU,CAAC;AAGzG,iBAAW,QAAQ,OAAO;AACxB,cAAM,IAAI,aAAa,IAAI;AAC3B,YAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,QAAQ,SAAS,MAAM,SAAS,aAAa,GAAG;AACvF,iBAAO,IAAI,CAAC;AACZ,cAAI,KAAK,sBAAsB,MAAO,OAAM,QAAQ,IAAI;AAAA,cACnD,OAAM,KAAK,IAAI;AAAA,QACtB;AAAA,MACF;AAAA,IACF,CAAC,CAAC;AAAA,EACJ;AAEA,QAAM,cAAc,KAAK,eAAe,QAAQ;AAEhD,SAAO,MAAM,SAAS,KAAK,QAAQ,SAAS,YAAY;AACtD,UAAM,QAAQ,MAAM,OAAO,GAAG,WAAW;AACzC,UAAM,WAAW,KAAK;AAAA,EACxB;AAGA,MAAI;AACJ,MAAI;AAEJ,QAAM,UAAU,KAAK,iBAAiB,CAAC;AACvC,MAAI,QAAQ,SAAS,WAAW,GAAG;AACjC,gBAAY,QAAQ,IAAI,OAAK,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,UAAU,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACpI;AACA,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS;AACf,UAAM,OAAO,QAAQ;AAAA,MAAI,OACvB,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,SAAS,eAAe,IAAI,EAAE,SAAS,WAAW,IAAI,EAAE,SAAS,YAAY,KAAK,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,OAAK,IAAI,OAAO,CAAC,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG;AAAA,IAC/N;AACA,UAAM,CAAC,QAAQ,GAAG,IAAI,EAAE,KAAK,IAAI;AAAA,EACnC;AAEA,QAAM,aAAa,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,OAAO,QAAQ;AAAA,MACf,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC,cAAc,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/incremental-diff.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+export interface ContentHash {
+    url: string;
+    hash: string;
+    textLength: number;
+    computedAt: string;
+}
+export interface DiffResult {
+    url: string;
+    changed: boolean;
+    previousHash?: string;
+    currentHash: string;
+    reason: 'new' | 'changed' | 'unchanged';
+}
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export declare function hashContent(text: string): string;
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export declare function normalizeText(text: string): string;
+export declare function computeContentHash(url: string, markdownText: string): ContentHash;
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export declare function diffContent(url: string, currentMarkdown: string, priorHashes: Map<string, string>): DiffResult;
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export declare function buildPriorHashMap(screenRecords: Array<{
+    url: string;
+    elementHash?: string | null;
+}>): Map<string, string>;
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export declare function partitionByChange(urls: string[], priorHashes: Map<string, string>): {
+    toProcess: string[];
+    toSkip: string[];
+};
+/** A single line-level change in text content (markdown, html, summary). */
+export interface TextChange {
+    type: 'added' | 'removed';
+    line: number;
+    content: string;
+}
+/** A change to a specific JSON path between two scraped JSON objects. */
+export interface JsonChange {
+    path: string;
+    old: unknown;
+    new: unknown;
+}
+/** Set-based change for collections like links or images. */
+export interface SetChange {
+    added: string[];
+    removed: string[];
+}
+/** Screenshot pixel-level change. changed is percentage (0-100). */
+export interface ImageChange {
+    size: number;
+    changed: number;
+    mask?: string;
+}
+/** All possible diffs for a single monitor tick. */
+export interface MonitorDiffs {
+    markdown?: TextChange[];
+    html?: TextChange[];
+    json?: JsonChange[];
+    summary?: TextChange[];
+    links?: SetChange;
+    images?: SetChange;
+    branding?: JsonChange[];
+    screenshot?: ImageChange;
+}
+/**
+ * Compute line-level text diffs between two strings.
+ * Returns TextChange[] showing exactly which lines were added or removed.
+ * No external dependencies — pure JS Myers-inspired algorithm.
+ */
+export declare function computeTextDiffs(previous: string, current: string): TextChange[];
+/**
+ * Compute set-based diffs for URL collections (links, images).
+ */
+export declare function computeSetDiffs(previous: string[], current: string[]): SetChange;
+/**
+ * Flatten a JSON object into dot-notation path/value pairs.
+ * Used to compute JsonChange[] between two JSON extraction results.
+ */
+export declare function flattenJson(obj: unknown, prefix?: string): Record<string, unknown>;
+/**
+ * Compute JSON path-level diffs between two extraction results.
+ */
+export declare function computeJsonDiffs(previous: unknown, current: unknown): JsonChange[];
+/**
+ * Count total number of changes across all diff types.
+ */
+export declare function countMonitorDiffs(diffs: MonitorDiffs): number;