@detiq/crawler 0.1.3

package/dist/http-engine.js

@@ -0,0 +1,304 @@
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+import { parseHTML } from "linkedom";
+import { RobotsCache } from "./robots-txt.js";
+import { ResponseCache } from "./response-cache.js";
+import { extractMetadata, extractTables, extractImages } from "./extractors/advanced.js";
+import { CHROME_CIPHERS, CHROME_CURVES } from "./tls-fingerprint.js";
+import { renderingPredictor } from "./engine-router.js";
+let chromeFetchDispatcher;
+async function getChromeFetchDispatcher() {
+  if (!chromeFetchDispatcher) {
+    try {
+      const impitMod = await import("impit");
+      const ImpitClass = impitMod.Impit ?? impitMod.default;
+      if (ImpitClass) {
+        chromeFetchDispatcher = new ImpitClass({ browser: "chrome" });
+        return chromeFetchDispatcher;
+      }
+    } catch {
+    }
+    const { Agent } = await import("undici");
+    chromeFetchDispatcher = new Agent({
+      connect: {
+        ciphers: CHROME_CIPHERS,
+        ecdhCurve: CHROME_CURVES,
+        minVersion: "TLSv1.2",
+        maxVersion: "TLSv1.3"
+      },
+      keepAliveTimeout: 1e4,
+      keepAliveMaxTimeout: 3e4
+    });
+  }
+  return chromeFetchDispatcher;
+}
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+function canonicalize(url) {
+  try {
+    const u = new URL(url);
+    u.hash = "";
+    for (const p of ["utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term", "fbclid", "gclid", "ref", "referrer"]) {
+      u.searchParams.delete(p);
+    }
+    return u.href.toLowerCase().replace(/\/$/, "");
+  } catch {
+    return url.toLowerCase();
+  }
+}
+function extractLinks(html, baseUrl) {
+  const links = [];
+  const seen = /* @__PURE__ */ new Set();
+  try {
+    const base = new URL(baseUrl);
+    const { document } = parseHTML(html);
+    const anchors = Array.from(document.querySelectorAll("a[href]"));
+    for (const a of anchors) {
+      const href = a.getAttribute("href") ?? "";
+      if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) continue;
+      try {
+        const resolved = new URL(href, base).href;
+        if (new URL(resolved).origin !== base.origin) continue;
+        const c = canonicalize(resolved);
+        if (!seen.has(c)) {
+          seen.add(c);
+          links.push(resolved);
+        }
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return links;
+}
+function extractMarkdown(html, url) {
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll("script, style, noscript"))) tag.remove();
+    const reader = new Readability(document, { charThreshold: 50 });
+    const article = reader.parse();
+    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
+      const md = turndown.turndown(article.content ?? html);
+      return { markdown: md, title: article.title ?? null };
+    }
+  } catch {
+  }
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll("script, style, noscript, nav, footer"))) tag.remove();
+    const bodyHtml = document.body?.innerHTML ?? html;
+    const md = turndown.turndown(bodyHtml);
+    const titleEl = document.querySelector("title");
+    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
+  } catch {
+    return { markdown: "", title: null };
+  }
+}
+async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
+  const t0 = performance.now();
+  if (cache.isFresh(url)) {
+    const cached = cache.get(url);
+    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
+  }
+  const validationHeaders = cache.getValidationHeaders(url);
+  const headers = {
+    "User-Agent": opts.userAgent,
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    // Chrome-like security headers reduce bot-detection fingerprint
+    "sec-ch-ua": '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Windows"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    ...validationHeaders,
+    ...opts.extraHeaders ?? {}
+  };
+  let response;
+  try {
+    const dispatcher = await getChromeFetchDispatcher();
+    response = await fetch(url, {
+      headers,
+      signal: AbortSignal.timeout(opts.timeoutMs),
+      redirect: "follow",
+      dispatcher
+    });
+  } catch (e) {
+    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
+  }
+  const durationMs = performance.now() - t0;
+  if (response.status === 304) {
+    const cached = cache.get(url);
+    if (cached) {
+      cache.markHit(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
+    }
+  }
+  if (response.status === 429 || response.status === 503) {
+    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    const retryAfter = parseInt(response.headers.get("retry-after") ?? "0") || 0;
+    const baseDelay = retryAfter > 0 ? retryAfter * 1e3 : Math.min(6e4, 1e3 * Math.pow(2, retryCount));
+    const jitter = Math.random() * 1e3;
+    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    await new Promise((r) => setTimeout(r, baseDelay + jitter));
+    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+  }
+  if (response.status >= 500) {
+    if (retryCount < 3) {
+      await new Promise((r) => setTimeout(r, 2e3 * (retryCount + 1)));
+      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    }
+    if (cache.staleIfError(url)) {
+      const cached = cache.get(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
+    }
+  }
+  const contentType = response.headers.get("content-type") ?? "";
+  const body = await response.text();
+  const etag = response.headers.get("etag") ?? void 0;
+  const lastModified = response.headers.get("last-modified") ?? void 0;
+  const cacheControl = response.headers.get("cache-control") ?? void 0;
+  if (response.ok && contentType.includes("html")) {
+    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
+  }
+  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
+}
+async function httpCrawl(opts) {
+  const startTime = performance.now();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? "/tmp/zeta-cache";
+  const timeoutMs = opts.timeoutMs ?? 15e3;
+  const userAgent = opts.userAgent ?? "ZeTa-Crawler/1.0 (+https://zetaai.dpdns.org)";
+  const cache = new ResponseCache(cacheDir);
+  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
+  const urlFilterRe = opts.urlFilter ? (() => {
+    try {
+      return new RegExp(opts.urlFilter, "i");
+    } catch {
+      return null;
+    }
+  })() : null;
+  const visited = /* @__PURE__ */ new Set();
+  const queue = [opts.appUrl];
+  const queued = /* @__PURE__ */ new Set([canonicalize(opts.appUrl)]);
+  const screens = [];
+  let cachedCount = 0;
+  let errorCount = 0;
+  let rateLimitedCount = 0;
+  const batchFetch = async (urls) => {
+    await Promise.all(urls.map(async (url) => {
+      const canonUrl = canonicalize(url);
+      if (visited.has(canonUrl)) return;
+      visited.add(canonUrl);
+      if (robots && !await robots.isAllowed(url).catch(() => true)) {
+        console.log(`[http-engine] robots.txt blocked: ${url}`);
+        return;
+      }
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) return;
+      } catch {
+      }
+      if (urlFilterRe) {
+        try {
+          const { pathname, search } = new URL(url);
+          if (!urlFilterRe.test(pathname + search)) {
+            console.log(`[http-engine] filtered out: ${url}`);
+            return;
+          }
+        } catch {
+        }
+      }
+      let fetchResult;
+      try {
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
+      } catch (err) {
+        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
+        errorCount++;
+        return;
+      }
+      if (!fetchResult.contentType.includes("html")) return;
+      if (fetchResult.statusCode >= 400) return;
+      if (fetchResult.fromCache) cachedCount++;
+      const { markdown, title } = extractMarkdown(fetchResult.body, url);
+      const fullMeta = extractMetadata(fetchResult.body, url);
+      const links = extractLinks(fetchResult.body, url);
+      const tables = extractTables(fetchResult.body);
+      const images = extractImages(fetchResult.body, url);
+      const screen = {
+        url,
+        title: title ?? fullMeta.title,
+        markdown,
+        html: opts.includeRawHtml ? fetchResult.body : void 0,
+        metadata: {
+          description: fullMeta.description,
+          ogTitle: fullMeta.ogTitle,
+          ogDescription: fullMeta.ogDescription,
+          ogImage: fullMeta.ogImage,
+          canonical: fullMeta.canonical,
+          schemaTypes: fullMeta.schemaTypes,
+          lang: fullMeta.lang
+        },
+        tables,
+        links,
+        images: images.map((i) => ({ src: i.src, alt: i.alt })),
+        statusCode: fetchResult.statusCode,
+        fromCache: fetchResult.fromCache,
+        requiresBrowser: renderingPredictor.getCached(url) === "browser",
+        durationMs: fetchResult.durationMs
+      };
+      screens.push(screen);
+      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
+      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
+      for (const link of links) {
+        const c = canonicalize(link);
+        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
+          queued.add(c);
+          if (opts.traversalStrategy === "DFS") queue.unshift(link);
+          else queue.push(link);
+        }
+      }
+    }));
+  };
+  const concurrency = opts.concurrency ?? 5;
+  while (queue.length > 0 && screens.length < maxScreens) {
+    const batch = queue.splice(0, concurrency);
+    await batchFetch(batch);
+  }
+  let jsonLines;
+  let csv;
+  const formats = opts.outputFormats ?? [];
+  if (formats.includes("jsonlines")) {
+    jsonLines = screens.map((s) => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join("\n");
+  }
+  if (formats.includes("csv")) {
+    const header = "url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs";
+    const rows = screens.map(
+      (s) => [s.url, s.title ?? "", s.metadata.description ?? "", s.metadata.ogTitle ?? "", s.metadata.schemaTypes.join("|"), s.statusCode, s.fromCache, Math.round(s.durationMs)].map((v) => `"${String(v).replace(/"/g, '""')}"`).join(",")
+    );
+    csv = [header, ...rows].join("\n");
+  }
+  const cacheStats = cache.stats();
+  return {
+    screens,
+    stats: {
+      total: screens.length,
+      cached: cachedCount,
+      errors: errorCount,
+      rateLimited: rateLimitedCount,
+      durationMs: performance.now() - startTime,
+      cacheHitRate: cacheStats.hitRate
+    },
+    jsonLines,
+    csv
+  };
+}
+export {
+  httpCrawl
+};
+//# sourceMappingURL=http-engine.js.map

package/dist/http-engine.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/http-engine.ts"],
+  "sourcesContent": ["/**\n * HTTP-only crawl engine \u2014 10-50x faster than browser for SSR/static sites.\n *\n * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.\n * No browser overhead. Falls back to browser engine on JS-heavy sites.\n *\n * Features:\n *   - HTTP/2 native (Node 20 built-in fetch)\n *   - ETag / Last-Modified response caching\n *   - 429 / 503 exponential backoff with jitter\n *   - BFS / DFS traversal\n *   - URL regex filtering\n *   - Robots.txt compliance\n *   - Link discovery (href, action, src)\n *   - Markdown extraction (Readability + Turndown)\n *   - OG / schema.org / meta extraction\n *   - Table \u2192 JSON extraction\n *   - JSON Lines, CSV, raw HTML output formats\n *   - Streaming progress via onProgress callback\n */\n\nimport { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { parseHTML } from 'linkedom';\nimport { RobotsCache } from './robots-txt.js';\nimport { ResponseCache } from './response-cache.js';\nimport { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';\nimport { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';\nimport { renderingPredictor } from './engine-router.js';\n\n// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback\nlet chromeFetchDispatcher: any;\nasync function getChromeFetchDispatcher() {\n  if (!chromeFetchDispatcher) {\n    try {\n      // impit provides native Chrome BoringSSL TLS fingerprint \u2014 true JA3/JA4 match\n      // @ts-ignore \u2014 impit is an optional dependency\n      const impitMod = await import('impit');\n      const ImpitClass = impitMod.Impit ?? impitMod.default;\n      if (ImpitClass) {\n        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });\n        return chromeFetchDispatcher;\n      }\n    } catch { /* impit not installed \u2014 fall through to undici */ }\n    const { Agent } = await import('undici');\n    chromeFetchDispatcher = new Agent({\n      connect: {\n        ciphers: CHROME_CIPHERS,\n        ecdhCurve: CHROME_CURVES,\n        minVersion: 'TLSv1.2' as any,\n        maxVersion: 'TLSv1.3' as any,\n      },\n      keepAliveTimeout: 10_000,\n      keepAliveMaxTimeout: 30_000,\n    });\n  }\n  return chromeFetchDispatcher;\n}\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\n\n// Pages unlikely to have testable UI \u2014 skip from HTTP crawl queue\nconst SKIP_PATH = /\\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;\n\nexport type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';\n\nexport interface HttpScreen {\n  url: string;\n  title: string | null;\n  markdown: string;\n  html?: string;\n  metadata: {\n    description: string | null;\n    ogTitle: string | null;\n    ogDescription: string | null;\n    ogImage: string | null;\n    canonical: string | null;\n    schemaTypes: string[];\n    lang: string | null;\n  };\n  tables: Array<Record<string, string>[]>;\n  links: string[];\n  images: Array<{ src: string; alt: string }>;\n  statusCode: number;\n  fromCache: boolean;\n  requiresBrowser?: boolean;\n  durationMs: number;\n}\n\nexport interface HttpCrawlOpts {\n  appUrl: string;\n  maxScreens?: number;            // default 100\n  urlFilter?: string;             // regex pattern \u2014 only crawl matching URLs\n  traversalStrategy?: 'BFS' | 'DFS'; // default BFS\n  outputFormats?: HttpOutputFormat[];\n  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'\n  userAgent?: string;\n  headers?: Record<string, string>;\n  respectRobotsTxt?: boolean;     // default true\n  includeRawHtml?: boolean;\n  timeoutMs?: number;             // per-request timeout, default 15000\n  concurrency?: number;           // default 5 parallel fetches\n  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;\n}\n\nexport interface HttpCrawlResult {\n  screens: HttpScreen[];\n  stats: {\n    total: number;\n    cached: number;\n    errors: number;\n    rateLimited: number;\n    durationMs: number;\n    cacheHitRate: number;\n  };\n  jsonLines?: string;\n  csv?: string;\n}\n\nfunction canonicalize(url: string): string {\n  try {\n    const u = new URL(url);\n    u.hash = '';\n    // Strip tracking params\n    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {\n      u.searchParams.delete(p);\n    }\n    return u.href.toLowerCase().replace(/\\/$/, '');\n  } catch {\n    return url.toLowerCase();\n  }\n}\n\nfunction extractLinks(html: string, baseUrl: string): string[] {\n  const links: string[] = [];\n  const seen = new Set<string>();\n  try {\n    const base = new URL(baseUrl);\n    const { document } = parseHTML(html);\n    const anchors = Array.from(document.querySelectorAll('a[href]'));\n    for (const a of anchors) {\n      const href = (a as any).getAttribute('href') ?? '';\n      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;\n      try {\n        const resolved = new URL(href, base).href;\n        // Same origin only\n        if (new URL(resolved).origin !== base.origin) continue;\n        const c = canonicalize(resolved);\n        if (!seen.has(c)) { seen.add(c); links.push(resolved); }\n      } catch { /* skip */ }\n    }\n  } catch { /* non-fatal */ }\n  return links;\n}\n\nfunction extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {\n  try {\n    const { document } = parseHTML(html);\n    // Remove script/style tags before Readability\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();\n    const reader = new Readability(document as any, { charThreshold: 50 });\n    const article = reader.parse();\n    if (article && article.textContent && article.textContent.split(/\\s+/).length >= 30) {\n      const md = turndown.turndown(article.content ?? html);\n      return { markdown: md, title: article.title ?? null };\n    }\n  } catch { /* fall through */ }\n  // Full-page fallback\n  try {\n    const { document } = parseHTML(html);\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();\n    const bodyHtml = (document as any).body?.innerHTML ?? html;\n    const md = turndown.turndown(bodyHtml);\n    const titleEl = (document as any).querySelector('title');\n    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };\n  } catch {\n    return { markdown: '', title: null };\n  }\n}\n\ninterface FetchResult {\n  body: string;\n  statusCode: number;\n  contentType: string;\n  fromCache: boolean;\n  durationMs: number;\n  etag?: string;\n  lastModified?: string;\n}\n\nasync function fetchWithBackoff(\n  url: string,\n  cache: ResponseCache,\n  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },\n  retryCount = 0\n): Promise<FetchResult> {\n  const t0 = performance.now();\n\n  if (cache.isFresh(url)) {\n    const cached = cache.get(url)!;\n    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };\n  }\n\n  const validationHeaders = cache.getValidationHeaders(url);\n\n  const headers: Record<string, string> = {\n    'User-Agent': opts.userAgent,\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n    'Accept-Language': 'en-US,en;q=0.9',\n    'Accept-Encoding': 'gzip, deflate, br, zstd',\n    // Chrome-like security headers reduce bot-detection fingerprint\n    'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n    'sec-ch-ua-mobile': '?0',\n    'sec-ch-ua-platform': '\"Windows\"',\n    'Sec-Fetch-Dest': 'document',\n    'Sec-Fetch-Mode': 'navigate',\n    'Sec-Fetch-Site': 'none',\n    'Sec-Fetch-User': '?1',\n    'Upgrade-Insecure-Requests': '1',\n    ...validationHeaders,\n    ...(opts.extraHeaders ?? {}),\n  };\n\n  let response: Response;\n  try {\n    const dispatcher = await getChromeFetchDispatcher();\n    response = await (fetch as any)(url, {\n      headers,\n      signal: AbortSignal.timeout(opts.timeoutMs),\n      redirect: 'follow',\n      dispatcher,\n    });\n  } catch (e: any) {\n    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);\n  }\n\n  const durationMs = performance.now() - t0;\n\n  // 304 Not Modified \u2014 use cached body\n  if (response.status === 304) {\n    const cached = cache.get(url);\n    if (cached) {\n      cache.markHit(url);\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };\n    }\n  }\n\n  // 429 / 503 \u2014 exponential backoff with jitter\n  if (response.status === 429 || response.status === 503) {\n    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);\n    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;\n    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));\n    const jitter = Math.random() * 1000;\n    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);\n    await new Promise(r => setTimeout(r, baseDelay + jitter));\n    return fetchWithBackoff(url, cache, opts, retryCount + 1);\n  }\n\n  // 5xx \u2014 retry up to 3 times\n  if (response.status >= 500) {\n    if (retryCount < 3) {\n      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));\n      return fetchWithBackoff(url, cache, opts, retryCount + 1);\n    }\n    if (cache.staleIfError(url)) {\n      const cached = cache.get(url)!;\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };\n    }\n  }\n\n  const contentType = response.headers.get('content-type') ?? '';\n  const body = await response.text();\n  const etag = response.headers.get('etag') ?? undefined;\n  const lastModified = response.headers.get('last-modified') ?? undefined;\n  const cacheControl = response.headers.get('cache-control') ?? undefined;\n\n  // Cache successful HTML responses\n  if (response.ok && contentType.includes('html')) {\n    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });\n  }\n\n  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };\n}\n\nexport async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {\n  const startTime = performance.now();\n  const maxScreens = opts.maxScreens ?? 100;\n  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';\n  const timeoutMs = opts.timeoutMs ?? 15_000;\n  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zetaai.dpdns.org)';\n  const cache = new ResponseCache(cacheDir);\n  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;\n  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;\n\n  const visited = new Set<string>();\n  const queue: string[] = [opts.appUrl];\n  const queued = new Set<string>([canonicalize(opts.appUrl)]);\n  const screens: HttpScreen[] = [];\n  let cachedCount = 0;\n  let errorCount = 0;\n  let rateLimitedCount = 0;\n\n  const batchFetch = async (urls: string[]): Promise<void> => {\n    await Promise.all(urls.map(async (url) => {\n      const canonUrl = canonicalize(url);\n      if (visited.has(canonUrl)) return;\n      visited.add(canonUrl);\n\n      // Robots.txt check\n      if (robots && !(await robots.isAllowed(url).catch(() => true))) {\n        console.log(`[http-engine] robots.txt blocked: ${url}`);\n        return;\n      }\n\n      // Skip content/legal pages\n      try {\n        const { pathname } = new URL(url);\n        if (SKIP_PATH.test(pathname)) return;\n      } catch { /* skip */ }\n\n      // URL filter\n      if (urlFilterRe) {\n        try {\n          const { pathname, search } = new URL(url);\n          if (!urlFilterRe.test(pathname + search)) {\n            console.log(`[http-engine] filtered out: ${url}`);\n            return;\n          }\n        } catch { /* skip */ }\n      }\n\n      let fetchResult: FetchResult;\n      try {\n        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });\n        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;\n      } catch (err: any) {\n        console.error(`[http-engine] error fetching ${url}: ${err.message}`);\n        errorCount++;\n        return;\n      }\n\n      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML\n      if (fetchResult.statusCode >= 400) return;\n      if (fetchResult.fromCache) cachedCount++;\n\n      const { markdown, title } = extractMarkdown(fetchResult.body, url);\n      const fullMeta = extractMetadata(fetchResult.body, url);\n      const links = extractLinks(fetchResult.body, url);\n      const tables = extractTables(fetchResult.body);\n      const images = extractImages(fetchResult.body, url);\n\n      const screen: HttpScreen = {\n        url,\n        title: title ?? fullMeta.title,\n        markdown,\n        html: opts.includeRawHtml ? fetchResult.body : undefined,\n        metadata: {\n          description: fullMeta.description,\n          ogTitle: fullMeta.ogTitle,\n          ogDescription: fullMeta.ogDescription,\n          ogImage: fullMeta.ogImage,\n          canonical: fullMeta.canonical,\n          schemaTypes: fullMeta.schemaTypes,\n          lang: fullMeta.lang,\n        },\n        tables,\n        links,\n        images: images.map(i => ({ src: i.src, alt: i.alt })),\n        statusCode: fetchResult.statusCode,\n        fromCache: fetchResult.fromCache,\n        requiresBrowser: renderingPredictor.getCached(url) === 'browser',\n        durationMs: fetchResult.durationMs,\n      };\n      screens.push(screen);\n\n      // Record rendering prediction: SPA signals \u2192 browser needed; content-rich \u2192 http engine works\n      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);\n\n      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });\n\n      // Enqueue discovered links\n      for (const link of links) {\n        const c = canonicalize(link);\n        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {\n          queued.add(c);\n          if (opts.traversalStrategy === 'DFS') queue.unshift(link);\n          else queue.push(link);\n        }\n      }\n    }));\n  };\n\n  const concurrency = opts.concurrency ?? 5;\n\n  while (queue.length > 0 && screens.length < maxScreens) {\n    const batch = queue.splice(0, concurrency);\n    await batchFetch(batch);\n  }\n\n  // Output formats\n  let jsonLines: string | undefined;\n  let csv: string | undefined;\n\n  const formats = opts.outputFormats ?? [];\n  if (formats.includes('jsonlines')) {\n    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\\n');\n  }\n  if (formats.includes('csv')) {\n    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';\n    const rows = screens.map(s =>\n      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `\"${String(v).replace(/\"/g, '\"\"')}\"`).join(',')\n    );\n    csv = [header, ...rows].join('\\n');\n  }\n\n  const cacheStats = cache.stats();\n  return {\n    screens,\n    stats: {\n      total: screens.length,\n      cached: cachedCount,\n      errors: errorCount,\n      rateLimited: rateLimitedCount,\n      durationMs: performance.now() - startTime,\n      cacheHitRate: cacheStats.hitRate,\n    },\n    jsonLines,\n    csv,\n  };\n}\n"],
+  "mappings": "AAwBA,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB,eAAe,qBAAqB;AAC9D,SAAS,gBAAgB,qBAAqB;AAC9C,SAAS,0BAA0B;AAGnC,IAAI;AACJ,eAAe,2BAA2B;AACxC,MAAI,CAAC,uBAAuB;AAC1B,QAAI;AAGF,YAAM,WAAW,MAAM,OAAO,OAAO;AACrC,YAAM,aAAa,SAAS,SAAS,SAAS;AAC9C,UAAI,YAAY;AACd,gCAAwB,IAAI,WAAW,EAAE,SAAS,SAAS,CAAC;AAC5D,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAqD;AAC7D,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,4BAAwB,IAAI,MAAM;AAAA,MAChC,SAAS;AAAA,QACP,SAAS;AAAA,QACT,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,MACd;AAAA,MACA,kBAAkB;AAAA,MAClB,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AAGtF,MAAM,YAAY;AAyDlB,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AAET,eAAW,KAAK,CAAC,cAAc,cAAc,gBAAgB,eAAe,YAAY,UAAU,SAAS,OAAO,UAAU,GAAG;AAC7H,QAAE,aAAa,OAAO,CAAC;AAAA,IACzB;AACA,WAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,OAAO,EAAE;AAAA,EAC/C,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;AAEA,SAAS,aAAa,MAAc,SAA2B;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC;AAC/D,eAAW,KAAK,SAAS;AACvB,YAAM,OAAQ,EAAU,aAAa,MAAM,KAAK;AAChD,UAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,EAAG;AAC9H,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,MAAM,IAAI,EAAE;AAErC,YAAI,IAAI,IAAI,QAAQ,EAAE,WAAW,KAAK,OAAQ;AAC9C,cAAM,IAAI,aAAa,QAAQ;AAC/B,YAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAAE,eAAK,IAAI,CAAC;AAAG,gBAAM,KAAK,QAAQ;AAAA,QAAG;AAAA,MACzD,QAAQ;AAAA,MAAa;AAAA,IACvB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,SAAS,gBAAgB,MAAc,KAAyD;AAC9F,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,yBAAyB,CAAC,EAAG,CAAC,IAAY,OAAO;AACxG,UAAM,SAAS,IAAI,YAAY,UAAiB,EAAE,eAAe,GAAG,CAAC;AACrE,UAAM,UAAU,OAAO,MAAM;AAC7B,QAAI,WAAW,QAAQ,eAAe,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,IAAI;AACnF,YAAM,KAAK,SAAS,SAAS,QAAQ,WAAW,IAAI;AACpD,aAAO,EAAE,UAAU,IAAI,OAAO,QAAQ,SAAS,KAAK;AAAA,IACtD;AAAA,EACF,QAAQ;AAAA,EAAqB;AAE7B,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,sCAAsC,CAAC,EAAG,CAAC,IAAY,OAAO;AACrH,UAAM,WAAY,SAAiB,MAAM,aAAa;AACtD,UAAM,KAAK,SAAS,SAAS,QAAQ;AACrC,UAAM,UAAW,SAAiB,cAAc,OAAO;AACvD,WAAO,EAAE,UAAU,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,KAAK;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,UAAU,IAAI,OAAO,KAAK;AAAA,EACrC;AACF;AAYA,eAAe,iBACb,KACA,OACA,MACA,aAAa,GACS;AACtB,QAAM,KAAK,YAAY,IAAI;AAE3B,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,WAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,EAAE;AAAA,EAC/G;AAEA,QAAM,oBAAoB,MAAM,qBAAqB,GAAG;AAExD,QAAM,UAAkC;AAAA,IACtC,cAAc,KAAK;AAAA,IACnB,UAAU;AAAA,IACV,mBAAmB;AAAA,IACnB,mBAAmB;AAAA;AAAA,IAEnB,aAAa;AAAA,IACb,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,6BAA6B;AAAA,IAC7B,GAAG;AAAA,IACH,GAAI,KAAK,gBAAgB,CAAC;AAAA,EAC5B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,MAAM,yBAAyB;AAClD,eAAW,MAAO,MAAc,KAAK;AAAA,MACnC;AAAA,MACA,QAAQ,YAAY,QAAQ,KAAK,SAAS;AAAA,MAC1C,UAAU;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH,SAAS,GAAQ;AACf,UAAM,IAAI,MAAM,kCAAkC,GAAG,KAAK,EAAE,OAAO,EAAE;AAAA,EACvE;AAEA,QAAM,aAAa,YAAY,IAAI,IAAI;AAGvC,MAAI,SAAS,WAAW,KAAK;AAC3B,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV,YAAM,QAAQ,GAAG;AACjB,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,WAAW;AAAA,IAC5G;AAAA,EACF;AAGA,MAAI,SAAS,WAAW,OAAO,SAAS,WAAW,KAAK;AACtD,QAAI,cAAc,EAAG,OAAM,IAAI,MAAM,+CAA+C,GAAG,EAAE;AACzF,UAAM,aAAa,SAAS,SAAS,QAAQ,IAAI,aAAa,KAAK,GAAG,KAAK;AAC3E,UAAM,YAAY,aAAa,IAAI,aAAa,MAAO,KAAK,IAAI,KAAQ,MAAQ,KAAK,IAAI,GAAG,UAAU,CAAC;AACvG,UAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAQ,IAAI,iBAAiB,SAAS,MAAM,OAAO,GAAG,uBAAkB,KAAK,MAAM,YAAY,MAAM,CAAC,aAAa,aAAa,CAAC,KAAK;AACtI,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,YAAY,MAAM,CAAC;AACxD,WAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,EAC1D;AAGA,MAAI,SAAS,UAAU,KAAK;AAC1B,QAAI,aAAa,GAAG;AAClB,YAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,OAAS,aAAa,EAAE,CAAC;AAC9D,aAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,IAC1D;AACA,QAAI,MAAM,aAAa,GAAG,GAAG;AAC3B,YAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,YAAY,IAAI,IAAI,GAAG;AAAA,IACpI;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAM,OAAO,MAAM,SAAS,KAAK;AACjC,QAAM,OAAO,SAAS,QAAQ,IAAI,MAAM,KAAK;AAC7C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAC9D,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAG9D,MAAI,SAAS,MAAM,YAAY,SAAS,MAAM,GAAG;AAC/C,UAAM,IAAI,KAAK,EAAE,KAAK,MAAM,cAAc,cAAc,MAAM,aAAa,YAAY,SAAS,OAAO,CAAC;AAAA,EAC1G;AAEA,SAAO,EAAE,MAAM,YAAY,SAAS,QAAQ,aAAa,WAAW,OAAO,YAAY,MAAM,aAAa;AAC5G;AAEA,eAAsB,UAAU,MAA+C;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,WAAW,KAAK,YAAY,QAAQ,IAAI,qBAAqB;AACnE,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,QAAQ,IAAI,cAAc,QAAQ;AACxC,QAAM,SAAS,KAAK,qBAAqB,QAAQ,IAAI,YAAY,IAAI;AACrE,QAAM,cAAc,KAAK,aAAa,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,OAAO,KAAK,WAAY,GAAG;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAM;AAAA,EAAE,GAAG,IAAI;AAE7H,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,QAAkB,CAAC,KAAK,MAAM;AACpC,QAAM,SAAS,oBAAI,IAAY,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;AAC1D,QAAM,UAAwB,CAAC;AAC/B,MAAI,cAAc;AAClB,MAAI,aAAa;AACjB,MAAI,mBAAmB;AAEvB,QAAM,aAAa,OAAO,SAAkC;AAC1D,UAAM,QAAQ,IAAI,KAAK,IAAI,OAAO,QAAQ;AACxC,YAAM,WAAW,aAAa,GAAG;AACjC,UAAI,QAAQ,IAAI,QAAQ,EAAG;AAC3B,cAAQ,IAAI,QAAQ;AAGpB,UAAI,UAAU,CAAE,MAAM,OAAO,UAAU,GAAG,EAAE,MAAM,MAAM,IAAI,GAAI;AAC9D,gBAAQ,IAAI,qCAAqC,GAAG,EAAE;AACtD;AAAA,MACF;AAGA,UAAI;AACF,cAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,YAAI,UAAU,KAAK,QAAQ,EAAG;AAAA,MAChC,QAAQ;AAAA,MAAa;AAGrB,UAAI,aAAa;AACf,YAAI;AACF,gBAAM,EAAE,UAAU,OAAO,IAAI,IAAI,IAAI,GAAG;AACxC,cAAI,CAAC,YAAY,KAAK,WAAW,MAAM,GAAG;AACxC,oBAAQ,IAAI,+BAA+B,GAAG,EAAE;AAChD;AAAA,UACF;AAAA,QACF,QAAQ;AAAA,QAAa;AAAA,MACvB;AAEA,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,iBAAiB,KAAK,OAAO,EAAE,WAAW,WAAW,cAAc,KAAK,QAAQ,CAAC;AACrG,YAAI,YAAY,eAAe,OAAO,YAAY,eAAe,IAAK;AAAA,MACxE,SAAS,KAAU;AACjB,gBAAQ,MAAM,gCAAgC,GAAG,KAAK,IAAI,OAAO,EAAE;AACnE;AACA;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,YAAY,SAAS,MAAM,EAAG;AAC/C,UAAI,YAAY,cAAc,IAAK;AACnC,UAAI,YAAY,UAAW;AAE3B,YAAM,EAAE,UAAU,MAAM,IAAI,gBAAgB,YAAY,MAAM,GAAG;AACjE,YAAM,WAAW,gBAAgB,YAAY,MAAM,GAAG;AACtD,YAAM,QAAQ,aAAa,YAAY,MAAM,GAAG;AAChD,YAAM,SAAS,cAAc,YAAY,IAAI;AAC7C,YAAM,SAAS,cAAc,YAAY,MAAM,GAAG;AAElD,YAAM,SAAqB;AAAA,QACzB;AAAA,QACA,OAAO,SAAS,SAAS;AAAA,QACzB;AAAA,QACA,MAAM,KAAK,iBAAiB,YAAY,OAAO;AAAA,QAC/C,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,SAAS,SAAS;AAAA,UAClB,eAAe,SAAS;AAAA,UACxB,SAAS,SAAS;AAAA,UAClB,WAAW,SAAS;AAAA,UACpB,aAAa,SAAS;AAAA,UACtB,MAAM,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE;AAAA,QACpD,YAAY,YAAY;AAAA,QACxB,WAAW,YAAY;AAAA,QACvB,iBAAiB,mBAAmB,UAAU,GAAG,MAAM;AAAA,QACvD,YAAY,YAAY;AAAA,MAC1B;AACA,cAAQ,KAAK,MAAM;AAGnB,yBAAmB,QAAQ,KAAK,YAAY,MAAM,YAAY,UAAU;AAExE,WAAK,aAAa,EAAE,KAAK,cAAc,QAAQ,QAAQ,OAAO,YAAY,QAAQ,YAAY,UAAU,CAAC;AAGzG,iBAAW,QAAQ,OAAO;AACxB,cAAM,IAAI,aAAa,IAAI;AAC3B,YAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,QAAQ,SAAS,MAAM,SAAS,aAAa,GAAG;AACvF,iBAAO,IAAI,CAAC;AACZ,cAAI,KAAK,sBAAsB,MAAO,OAAM,QAAQ,IAAI;AAAA,cACnD,OAAM,KAAK,IAAI;AAAA,QACtB;AAAA,MACF;AAAA,IACF,CAAC,CAAC;AAAA,EACJ;AAEA,QAAM,cAAc,KAAK,eAAe;AAExC,SAAO,MAAM,SAAS,KAAK,QAAQ,SAAS,YAAY;AACtD,UAAM,QAAQ,MAAM,OAAO,GAAG,WAAW;AACzC,UAAM,WAAW,KAAK;AAAA,EACxB;AAGA,MAAI;AACJ,MAAI;AAEJ,QAAM,UAAU,KAAK,iBAAiB,CAAC;AACvC,MAAI,QAAQ,SAAS,WAAW,GAAG;AACjC,gBAAY,QAAQ,IAAI,OAAK,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,UAAU,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACpI;AACA,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS;AACf,UAAM,OAAO,QAAQ;AAAA,MAAI,OACvB,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,SAAS,eAAe,IAAI,EAAE,SAAS,WAAW,IAAI,EAAE,SAAS,YAAY,KAAK,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,OAAK,IAAI,OAAO,CAAC,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG;AAAA,IAC/N;AACA,UAAM,CAAC,QAAQ,GAAG,IAAI,EAAE,KAAK,IAAI;AAAA,EACnC;AAEA,QAAM,aAAa,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,OAAO,QAAQ;AAAA,MACf,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC,cAAc,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/incremental-diff.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+export interface ContentHash {
+    url: string;
+    hash: string;
+    textLength: number;
+    computedAt: string;
+}
+export interface DiffResult {
+    url: string;
+    changed: boolean;
+    previousHash?: string;
+    currentHash: string;
+    reason: 'new' | 'changed' | 'unchanged';
+}
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export declare function hashContent(text: string): string;
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export declare function normalizeText(text: string): string;
+export declare function computeContentHash(url: string, markdownText: string): ContentHash;
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export declare function diffContent(url: string, currentMarkdown: string, priorHashes: Map<string, string>): DiffResult;
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export declare function buildPriorHashMap(screenRecords: Array<{
+    url: string;
+    elementHash?: string | null;
+}>): Map<string, string>;
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export declare function partitionByChange(urls: string[], priorHashes: Map<string, string>): {
+    toProcess: string[];
+    toSkip: string[];
+};

package/dist/incremental-diff.js

@@ -0,0 +1,61 @@
+function hashContent(text) {
+  let h1 = 5381;
+  let h2 = 52711;
+  for (let i = 0; i < text.length; i++) {
+    const c = text.charCodeAt(i);
+    h1 = (Math.imul(h1, 33) ^ c) >>> 0;
+    h2 = (Math.imul(h2, 31) ^ c) >>> 0;
+  }
+  return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
+}
+function normalizeText(text) {
+  return text.split("\n").map((l) => l.trim()).filter((l) => l.length > 0).join("\n");
+}
+function computeContentHash(url, markdownText) {
+  const normalized = normalizeText(markdownText);
+  return {
+    url,
+    hash: hashContent(normalized),
+    textLength: normalized.length,
+    computedAt: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+function diffContent(url, currentMarkdown, priorHashes) {
+  const current = computeContentHash(url, currentMarkdown);
+  const prior = priorHashes.get(url);
+  if (!prior) {
+    return { url, changed: true, currentHash: current.hash, reason: "new" };
+  }
+  if (prior === current.hash) {
+    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: "unchanged" };
+  }
+  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: "changed" };
+}
+function buildPriorHashMap(screenRecords) {
+  const map = /* @__PURE__ */ new Map();
+  for (const s of screenRecords) {
+    if (s.url && s.elementHash) map.set(s.url, s.elementHash);
+  }
+  return map;
+}
+function partitionByChange(urls, priorHashes) {
+  const toProcess = [];
+  const toSkip = [];
+  for (const url of urls) {
+    if (priorHashes.has(url)) {
+      toProcess.push(url);
+    } else {
+      toProcess.push(url);
+    }
+  }
+  return { toProcess, toSkip };
+}
+export {
+  buildPriorHashMap,
+  computeContentHash,
+  diffContent,
+  hashContent,
+  normalizeText,
+  partitionByChange
+};
+//# sourceMappingURL=incremental-diff.js.map

package/dist/incremental-diff.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/incremental-diff.ts"],
+  "sourcesContent": ["/**\n * Incremental crawl diff \u2014 compute stable content hashes and skip unchanged screens.\n *\n * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).\n * On the next crawl of the same project, pages with the same fingerprint are\n * marked SKIPPED instead of re-processed, drastically reducing crawl time and\n * LLM token spend for sites with mostly-static content.\n *\n * Design: \"same content hash\" means the meaningful text hasn't changed.\n * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.\n */\n\nexport interface ContentHash {\n  url: string;\n  hash: string;\n  textLength: number;\n  computedAt: string; // ISO timestamp\n}\n\nexport interface DiffResult {\n  url: string;\n  changed: boolean;\n  previousHash?: string;\n  currentHash: string;\n  reason: 'new' | 'changed' | 'unchanged';\n}\n\n/**\n * Compute a stable 64-bit-ish hash from text content.\n * djb2 variant \u2014 fast, no deps, collision-resistant enough for content fingerprinting.\n */\nexport function hashContent(text: string): string {\n  let h1 = 5381;\n  let h2 = 52711;\n  for (let i = 0; i < text.length; i++) {\n    const c = text.charCodeAt(i);\n    h1 = (Math.imul(h1, 33) ^ c) >>> 0;\n    h2 = (Math.imul(h2, 31) ^ c) >>> 0;\n  }\n  return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');\n}\n\n/**\n * Normalize text before hashing \u2014 strips whitespace-only lines, trims each line,\n * collapses runs of blank lines. Makes hash stable across minor formatting changes.\n */\nexport function normalizeText(text: string): string {\n  return text\n    .split('\\n')\n    .map(l => l.trim())\n    .filter(l => l.length > 0)\n    .join('\\n');\n}\n\nexport function computeContentHash(url: string, markdownText: string): ContentHash {\n  const normalized = normalizeText(markdownText);\n  return {\n    url,\n    hash: hashContent(normalized),\n    textLength: normalized.length,\n    computedAt: new Date().toISOString(),\n  };\n}\n\n/**\n * Compare a page's current content against prior hashes.\n * Returns { changed: false } for unchanged pages \u2014 caller should skip re-processing.\n */\nexport function diffContent(\n  url: string,\n  currentMarkdown: string,\n  priorHashes: Map<string, string>, // url \u2192 hash\n): DiffResult {\n  const current = computeContentHash(url, currentMarkdown);\n  const prior = priorHashes.get(url);\n\n  if (!prior) {\n    return { url, changed: true, currentHash: current.hash, reason: 'new' };\n  }\n  if (prior === current.hash) {\n    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };\n  }\n  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };\n}\n\n/**\n * Build a hash map from prior screen records (from DB query result).\n * screenRecords: Array<{ url: string; elementHash?: string | null }>\n */\nexport function buildPriorHashMap(screenRecords: Array<{ url: string; elementHash?: string | null }>): Map<string, string> {\n  const map = new Map<string, string>();\n  for (const s of screenRecords) {\n    if (s.url && s.elementHash) map.set(s.url, s.elementHash);\n  }\n  return map;\n}\n\n/**\n * Filter a batch of discovered URLs against prior hashes.\n * Returns { toProcess, toSkip } \u2014 caller enqueues toProcess, logs toSkip count.\n *\n * Note: filtering at URL level only works if the prior crawl already visited the URL.\n * For new URLs, they're always in toProcess.\n */\nexport function partitionByChange(\n  urls: string[],\n  priorHashes: Map<string, string>,\n): { toProcess: string[]; toSkip: string[] } {\n  const toProcess: string[] = [];\n  const toSkip: string[] = [];\n  for (const url of urls) {\n    if (priorHashes.has(url)) {\n      // Can't know if changed without fetching \u2014 still needs fetch, but hash after fetch\n      // will determine skip. Include all \u2014 hash check happens post-fetch.\n      toProcess.push(url);\n    } else {\n      toProcess.push(url);\n    }\n  }\n  return { toProcess, toSkip };\n}\n"],
+  "mappings": "AA+BO,SAAS,YAAY,MAAsB;AAChD,MAAI,KAAK;AACT,MAAI,KAAK;AACT,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,IAAI,KAAK,WAAW,CAAC;AAC3B,UAAM,KAAK,KAAK,IAAI,EAAE,IAAI,OAAO;AACjC,UAAM,KAAK,KAAK,IAAI,EAAE,IAAI,OAAO;AAAA,EACnC;AACA,UAAQ,OAAO,GAAG,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,KAAK,OAAO,GAAG,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAC3F;AAMO,SAAS,cAAc,MAAsB;AAClD,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,OAAK,EAAE,KAAK,CAAC,EACjB,OAAO,OAAK,EAAE,SAAS,CAAC,EACxB,KAAK,IAAI;AACd;AAEO,SAAS,mBAAmB,KAAa,cAAmC;AACjF,QAAM,aAAa,cAAc,YAAY;AAC7C,SAAO;AAAA,IACL;AAAA,IACA,MAAM,YAAY,UAAU;AAAA,IAC5B,YAAY,WAAW;AAAA,IACvB,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,EACrC;AACF;AAMO,SAAS,YACd,KACA,iBACA,aACY;AACZ,QAAM,UAAU,mBAAmB,KAAK,eAAe;AACvD,QAAM,QAAQ,YAAY,IAAI,GAAG;AAEjC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,KAAK,SAAS,MAAM,aAAa,QAAQ,MAAM,QAAQ,MAAM;AAAA,EACxE;AACA,MAAI,UAAU,QAAQ,MAAM;AAC1B,WAAO,EAAE,KAAK,SAAS,OAAO,cAAc,OAAO,aAAa,QAAQ,MAAM,QAAQ,YAAY;AAAA,EACpG;AACA,SAAO,EAAE,KAAK,SAAS,MAAM,cAAc,OAAO,aAAa,QAAQ,MAAM,QAAQ,UAAU;AACjG;AAMO,SAAS,kBAAkB,eAAyF;AACzH,QAAM,MAAM,oBAAI,IAAoB;AACpC,aAAW,KAAK,eAAe;AAC7B,QAAI,EAAE,OAAO,EAAE,YAAa,KAAI,IAAI,EAAE,KAAK,EAAE,WAAW;AAAA,EAC1D;AACA,SAAO;AACT;AASO,SAAS,kBACd,MACA,aAC2C;AAC3C,QAAM,YAAsB,CAAC;AAC7B,QAAM,SAAmB,CAAC;AAC1B,aAAW,OAAO,MAAM;AACtB,QAAI,YAAY,IAAI,GAAG,GAAG;AAGxB,gBAAU,KAAK,GAAG;AAAA,IACpB,OAAO;AACL,gBAAU,KAAK,GAAG;AAAA,IACpB;AAAA,EACF;AACA,SAAO,EAAE,WAAW,OAAO;AAC7B;",
+  "names": []
+}

package/dist/index.d.ts

@@ -0,0 +1,35 @@
+export * from './crawler.js';
+export * from './visual-diff-verdict.js';
+export * from './auth.js';
+export * from './screen-reader.js';
+export * from './job-queue.js';
+export { startSqsWorker } from './sqs-queue.js';
+export * from './session-capture.js';
+export * from './session-export.js';
+export * from './test-executor.js';
+export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
+export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
+export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
+export type { SsoConfig, SsoProvider } from './sso-token.js';
+export * from './engine-router.js';
+export * from './profiles/domain-profile.js';
+export * from './policies/crawl-safety.js';
+export * from './extractors/artifact-envelope.js';
+export * from './network-observer.js';
+export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
+export type { TlsFingerprintOptions } from './tls-fingerprint.js';
+export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
+export type { ExtensionSource, ExtensionProfile } from './browser-extension.js';
+export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
+export type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';
+export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
+export type { UrlCandidate } from './sitemap.js';
+export { httpCrawl } from './http-engine.js';
+export { extractMarkdown } from './markdown-extract.js';
+export type { ExtractedMarkdown } from './markdown-extract.js';
+export type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';
+export { TieredProxyManager } from './proxy.js';
+export type { ProxyTier, ProxyTierConfig } from './proxy.js';
+export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
+export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
+export type { Span, FinishedSpan } from './telemetry.js';

package/dist/index.js

@@ -0,0 +1,61 @@
+export * from "./crawler.js";
+export * from "./visual-diff-verdict.js";
+export * from "./auth.js";
+export * from "./screen-reader.js";
+export * from "./job-queue.js";
+import { startSqsWorker } from "./sqs-queue.js";
+export * from "./session-capture.js";
+export * from "./session-export.js";
+export * from "./test-executor.js";
+import { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from "./live-view.js";
+import { waitForEmailOTP, getInboxEmail } from "./email-otp.js";
+import { getSsoToken, getSsoClientCredentialsToken } from "./sso-token.js";
+export * from "./engine-router.js";
+export * from "./profiles/domain-profile.js";
+export * from "./policies/crawl-safety.js";
+export * from "./extractors/artifact-envelope.js";
+export * from "./network-observer.js";
+import { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from "./tls-fingerprint.js";
+import { prepareExtensionProfile, adBlockerProfile } from "./browser-extension.js";
+import { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from "./redis-queue.js";
+import { discoverProjectUrls, discoverSitemapUrls } from "./sitemap.js";
+import { httpCrawl } from "./http-engine.js";
+import { extractMarkdown } from "./markdown-extract.js";
+import { TieredProxyManager } from "./proxy.js";
+import { RedisCrawlFrontier, isRedisFrontierEnabled } from "./redis-frontier.js";
+import { startSpan, endSpan, withSpan, increment, flushMetrics } from "./telemetry.js";
+export {
+  CHROME_TLS_HEADERS,
+  RedisCrawlFrontier,
+  TieredProxyManager,
+  adBlockerProfile,
+  chromeTlsAgent,
+  closeLiveView,
+  completeLiveView,
+  createChromeTlsAgent,
+  createRedisQueue,
+  discoverProjectUrls,
+  discoverSitemapUrls,
+  endSpan,
+  extractMarkdown,
+  flushMetrics,
+  getFrame,
+  getInboxEmail,
+  getSsoClientCredentialsToken,
+  getSsoToken,
+  httpCrawl,
+  increment,
+  isRedisFrontierEnabled,
+  isRedisQueueEnabled,
+  liveViewNavigate,
+  maybeCreateRedisQueue,
+  prepareExtensionProfile,
+  probeTlsFingerprint,
+  sendInput,
+  startLiveView,
+  startSpan,
+  startSqsWorker,
+  waitForEmailOTP,
+  withSpan
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/index.ts"],
+  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\n"],
+  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;",
+  "names": []
+}

package/dist/input-fuzzer.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Input fuzzer — generate test data for form fields.
+ *
+ * For each field type, produces a set of test values covering:
+ * - Happy path (valid, normal input)
+ * - Boundary (min/max length, min/max value)
+ * - Edge cases (empty, very long, special chars, SQL injection probe, XSS probe)
+ * - Type-invalid (wrong type for the field)
+ *
+ * This feeds ZeTa's test generator to automatically create form validation tests.
+ * Values are safe probes (NOT actual exploits) — they test whether the app
+ * properly rejects/accepts boundary inputs.
+ */
+import type { FormField } from './form-detector.js';
+export type InputCase = {
+    label: string;
+    value: string;
+    expectedValid: boolean;
+    category: 'happy' | 'boundary' | 'edge' | 'invalid';
+};
+export declare function generateFieldCases(field: FormField): InputCase[];
+export interface FuzzedForm {
+    formIndex: number;
+    purpose: string;
+    fieldCases: Array<{
+        field: FormField;
+        cases: InputCase[];
+    }>;
+}
+export declare function fuzzForm(form: import('./form-detector.js').DetectedForm): FuzzedForm;