@detiq/crawler 0.1.7 → 0.1.9

package/dist/http-engine.js

@@ -172,7 +172,8 @@ async function httpCrawl(opts) {
   const maxScreens = opts.maxScreens ?? 100;
   const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? "/tmp/zeta-cache";
   const timeoutMs = opts.timeoutMs ?? 15e3;
-  const userAgent = opts.userAgent ?? "ZeTa-Crawler/1.0 (+https://zetaai.dpdns.org)";
+  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? "ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)";
+  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
   const urlFilterRe = opts.urlFilter ? (() => {

package/dist/http-engine.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/http-engine.ts"],
-  "sourcesContent": ["/**\n * HTTP-only crawl engine \u2014 10-50x faster than browser for SSR/static sites.\n *\n * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.\n * No browser overhead. Falls back to browser engine on JS-heavy sites.\n *\n * Features:\n *   - HTTP/2 native (Node 20 built-in fetch)\n *   - ETag / Last-Modified response caching\n *   - 429 / 503 exponential backoff with jitter\n *   - BFS / DFS traversal\n *   - URL regex filtering\n *   - Robots.txt compliance\n *   - Link discovery (href, action, src)\n *   - Markdown extraction (Readability + Turndown)\n *   - OG / schema.org / meta extraction\n *   - Table \u2192 JSON extraction\n *   - JSON Lines, CSV, raw HTML output formats\n *   - Streaming progress via onProgress callback\n */\n\nimport { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { parseHTML } from 'linkedom';\nimport { RobotsCache } from './robots-txt.js';\nimport { ResponseCache } from './response-cache.js';\nimport { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';\nimport { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';\nimport { renderingPredictor } from './engine-router.js';\n\n// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback\nlet chromeFetchDispatcher: any;\nasync function getChromeFetchDispatcher() {\n  if (!chromeFetchDispatcher) {\n    try {\n      // impit provides native Chrome BoringSSL TLS fingerprint \u2014 true JA3/JA4 match\n      // @ts-ignore \u2014 impit is an optional dependency\n      const impitMod = await import('impit');\n      const ImpitClass = impitMod.Impit ?? impitMod.default;\n      if (ImpitClass) {\n        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });\n        return chromeFetchDispatcher;\n      }\n    } catch { /* impit not installed \u2014 fall through to undici */ }\n    const { Agent } = await import('undici');\n    chromeFetchDispatcher = new Agent({\n      connect: {\n        ciphers: CHROME_CIPHERS,\n        ecdhCurve: CHROME_CURVES,\n        minVersion: 'TLSv1.2' as any,\n        maxVersion: 'TLSv1.3' as any,\n      },\n      keepAliveTimeout: 10_000,\n      keepAliveMaxTimeout: 30_000,\n    });\n  }\n  return chromeFetchDispatcher;\n}\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\n\n// Pages unlikely to have testable UI \u2014 skip from HTTP crawl queue\nconst SKIP_PATH = /\\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;\n\nexport type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';\n\nexport interface HttpScreen {\n  url: string;\n  title: string | null;\n  markdown: string;\n  html?: string;\n  metadata: {\n    description: string | null;\n    ogTitle: string | null;\n    ogDescription: string | null;\n    ogImage: string | null;\n    canonical: string | null;\n    schemaTypes: string[];\n    lang: string | null;\n  };\n  tables: Array<Record<string, string>[]>;\n  links: string[];\n  images: Array<{ src: string; alt: string }>;\n  statusCode: number;\n  fromCache: boolean;\n  requiresBrowser?: boolean;\n  durationMs: number;\n}\n\nexport interface HttpCrawlOpts {\n  appUrl: string;\n  maxScreens?: number;            // default 100\n  urlFilter?: string;             // regex pattern \u2014 only crawl matching URLs\n  traversalStrategy?: 'BFS' | 'DFS'; // default BFS\n  outputFormats?: HttpOutputFormat[];\n  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'\n  userAgent?: string;\n  headers?: Record<string, string>;\n  respectRobotsTxt?: boolean;     // default true\n  includeRawHtml?: boolean;\n  timeoutMs?: number;             // per-request timeout, default 15000\n  concurrency?: number;           // default 5 parallel fetches\n  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;\n}\n\nexport interface HttpCrawlResult {\n  screens: HttpScreen[];\n  stats: {\n    total: number;\n    cached: number;\n    errors: number;\n    rateLimited: number;\n    durationMs: number;\n    cacheHitRate: number;\n  };\n  jsonLines?: string;\n  csv?: string;\n}\n\nfunction canonicalize(url: string): string {\n  try {\n    const u = new URL(url);\n    u.hash = '';\n    // Strip tracking params\n    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {\n      u.searchParams.delete(p);\n    }\n    return u.href.toLowerCase().replace(/\\/$/, '');\n  } catch {\n    return url.toLowerCase();\n  }\n}\n\nfunction extractLinks(html: string, baseUrl: string): string[] {\n  const links: string[] = [];\n  const seen = new Set<string>();\n  try {\n    const base = new URL(baseUrl);\n    const { document } = parseHTML(html);\n    const anchors = Array.from(document.querySelectorAll('a[href]'));\n    for (const a of anchors) {\n      const href = (a as any).getAttribute('href') ?? '';\n      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;\n      try {\n        const resolved = new URL(href, base).href;\n        // Same origin only\n        if (new URL(resolved).origin !== base.origin) continue;\n        const c = canonicalize(resolved);\n        if (!seen.has(c)) { seen.add(c); links.push(resolved); }\n      } catch { /* skip */ }\n    }\n  } catch { /* non-fatal */ }\n  return links;\n}\n\nfunction extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {\n  try {\n    const { document } = parseHTML(html);\n    // Remove script/style tags before Readability\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();\n    const reader = new Readability(document as any, { charThreshold: 50 });\n    const article = reader.parse();\n    if (article && article.textContent && article.textContent.split(/\\s+/).length >= 30) {\n      const md = turndown.turndown(article.content ?? html);\n      return { markdown: md, title: article.title ?? null };\n    }\n  } catch { /* fall through */ }\n  // Full-page fallback\n  try {\n    const { document } = parseHTML(html);\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();\n    const bodyHtml = (document as any).body?.innerHTML ?? html;\n    const md = turndown.turndown(bodyHtml);\n    const titleEl = (document as any).querySelector('title');\n    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };\n  } catch {\n    return { markdown: '', title: null };\n  }\n}\n\ninterface FetchResult {\n  body: string;\n  statusCode: number;\n  contentType: string;\n  fromCache: boolean;\n  durationMs: number;\n  etag?: string;\n  lastModified?: string;\n}\n\nasync function fetchWithBackoff(\n  url: string,\n  cache: ResponseCache,\n  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },\n  retryCount = 0\n): Promise<FetchResult> {\n  const t0 = performance.now();\n\n  if (cache.isFresh(url)) {\n    const cached = cache.get(url)!;\n    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };\n  }\n\n  const validationHeaders = cache.getValidationHeaders(url);\n\n  const headers: Record<string, string> = {\n    'User-Agent': opts.userAgent,\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n    'Accept-Language': 'en-US,en;q=0.9',\n    'Accept-Encoding': 'gzip, deflate, br, zstd',\n    // Chrome-like security headers reduce bot-detection fingerprint\n    'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n    'sec-ch-ua-mobile': '?0',\n    'sec-ch-ua-platform': '\"Windows\"',\n    'Sec-Fetch-Dest': 'document',\n    'Sec-Fetch-Mode': 'navigate',\n    'Sec-Fetch-Site': 'none',\n    'Sec-Fetch-User': '?1',\n    'Upgrade-Insecure-Requests': '1',\n    ...validationHeaders,\n    ...(opts.extraHeaders ?? {}),\n  };\n\n  let response: Response;\n  try {\n    const dispatcher = await getChromeFetchDispatcher();\n    response = await (fetch as any)(url, {\n      headers,\n      signal: AbortSignal.timeout(opts.timeoutMs),\n      redirect: 'follow',\n      dispatcher,\n    });\n  } catch (e: any) {\n    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);\n  }\n\n  const durationMs = performance.now() - t0;\n\n  // 304 Not Modified \u2014 use cached body\n  if (response.status === 304) {\n    const cached = cache.get(url);\n    if (cached) {\n      cache.markHit(url);\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };\n    }\n  }\n\n  // 429 / 503 \u2014 exponential backoff with jitter\n  if (response.status === 429 || response.status === 503) {\n    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);\n    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;\n    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));\n    const jitter = Math.random() * 1000;\n    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);\n    await new Promise(r => setTimeout(r, baseDelay + jitter));\n    return fetchWithBackoff(url, cache, opts, retryCount + 1);\n  }\n\n  // 5xx \u2014 retry up to 3 times\n  if (response.status >= 500) {\n    if (retryCount < 3) {\n      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));\n      return fetchWithBackoff(url, cache, opts, retryCount + 1);\n    }\n    if (cache.staleIfError(url)) {\n      const cached = cache.get(url)!;\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };\n    }\n  }\n\n  const contentType = response.headers.get('content-type') ?? '';\n  const body = await response.text();\n  const etag = response.headers.get('etag') ?? undefined;\n  const lastModified = response.headers.get('last-modified') ?? undefined;\n  const cacheControl = response.headers.get('cache-control') ?? undefined;\n\n  // Cache successful HTML responses\n  if (response.ok && contentType.includes('html')) {\n    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });\n  }\n\n  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };\n}\n\nexport async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {\n  const startTime = performance.now();\n  const maxScreens = opts.maxScreens ?? 100;\n  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';\n  const timeoutMs = opts.timeoutMs ?? 15_000;\n  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zetaai.dpdns.org)';\n  const cache = new ResponseCache(cacheDir);\n  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;\n  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;\n\n  const visited = new Set<string>();\n  const queue: string[] = [opts.appUrl];\n  const queued = new Set<string>([canonicalize(opts.appUrl)]);\n  const screens: HttpScreen[] = [];\n  let cachedCount = 0;\n  let errorCount = 0;\n  let rateLimitedCount = 0;\n\n  const batchFetch = async (urls: string[]): Promise<void> => {\n    await Promise.all(urls.map(async (url) => {\n      const canonUrl = canonicalize(url);\n      if (visited.has(canonUrl)) return;\n      visited.add(canonUrl);\n\n      // Robots.txt check\n      if (robots && !(await robots.isAllowed(url).catch(() => true))) {\n        console.log(`[http-engine] robots.txt blocked: ${url}`);\n        return;\n      }\n\n      // Skip content/legal pages\n      try {\n        const { pathname } = new URL(url);\n        if (SKIP_PATH.test(pathname)) return;\n      } catch { /* skip */ }\n\n      // URL filter\n      if (urlFilterRe) {\n        try {\n          const { pathname, search } = new URL(url);\n          if (!urlFilterRe.test(pathname + search)) {\n            console.log(`[http-engine] filtered out: ${url}`);\n            return;\n          }\n        } catch { /* skip */ }\n      }\n\n      let fetchResult: FetchResult;\n      try {\n        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });\n        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;\n      } catch (err: any) {\n        console.error(`[http-engine] error fetching ${url}: ${err.message}`);\n        errorCount++;\n        return;\n      }\n\n      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML\n      if (fetchResult.statusCode >= 400) return;\n      if (fetchResult.fromCache) cachedCount++;\n\n      const { markdown, title } = extractMarkdown(fetchResult.body, url);\n      const fullMeta = extractMetadata(fetchResult.body, url);\n      const links = extractLinks(fetchResult.body, url);\n      const tables = extractTables(fetchResult.body);\n      const images = extractImages(fetchResult.body, url);\n\n      const screen: HttpScreen = {\n        url,\n        title: title ?? fullMeta.title,\n        markdown,\n        html: opts.includeRawHtml ? fetchResult.body : undefined,\n        metadata: {\n          description: fullMeta.description,\n          ogTitle: fullMeta.ogTitle,\n          ogDescription: fullMeta.ogDescription,\n          ogImage: fullMeta.ogImage,\n          canonical: fullMeta.canonical,\n          schemaTypes: fullMeta.schemaTypes,\n          lang: fullMeta.lang,\n        },\n        tables,\n        links,\n        images: images.map(i => ({ src: i.src, alt: i.alt })),\n        statusCode: fetchResult.statusCode,\n        fromCache: fetchResult.fromCache,\n        requiresBrowser: renderingPredictor.getCached(url) === 'browser',\n        durationMs: fetchResult.durationMs,\n      };\n      screens.push(screen);\n\n      // Record rendering prediction: SPA signals \u2192 browser needed; content-rich \u2192 http engine works\n      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);\n\n      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });\n\n      // Enqueue discovered links\n      for (const link of links) {\n        const c = canonicalize(link);\n        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {\n          queued.add(c);\n          if (opts.traversalStrategy === 'DFS') queue.unshift(link);\n          else queue.push(link);\n        }\n      }\n    }));\n  };\n\n  const concurrency = opts.concurrency ?? 5;\n\n  while (queue.length > 0 && screens.length < maxScreens) {\n    const batch = queue.splice(0, concurrency);\n    await batchFetch(batch);\n  }\n\n  // Output formats\n  let jsonLines: string | undefined;\n  let csv: string | undefined;\n\n  const formats = opts.outputFormats ?? [];\n  if (formats.includes('jsonlines')) {\n    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\\n');\n  }\n  if (formats.includes('csv')) {\n    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';\n    const rows = screens.map(s =>\n      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `\"${String(v).replace(/\"/g, '\"\"')}\"`).join(',')\n    );\n    csv = [header, ...rows].join('\\n');\n  }\n\n  const cacheStats = cache.stats();\n  return {\n    screens,\n    stats: {\n      total: screens.length,\n      cached: cachedCount,\n      errors: errorCount,\n      rateLimited: rateLimitedCount,\n      durationMs: performance.now() - startTime,\n      cacheHitRate: cacheStats.hitRate,\n    },\n    jsonLines,\n    csv,\n  };\n}\n"],
-  "mappings": "AAwBA,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB,eAAe,qBAAqB;AAC9D,SAAS,gBAAgB,qBAAqB;AAC9C,SAAS,0BAA0B;AAGnC,IAAI;AACJ,eAAe,2BAA2B;AACxC,MAAI,CAAC,uBAAuB;AAC1B,QAAI;AAGF,YAAM,WAAW,MAAM,OAAO,OAAO;AACrC,YAAM,aAAa,SAAS,SAAS,SAAS;AAC9C,UAAI,YAAY;AACd,gCAAwB,IAAI,WAAW,EAAE,SAAS,SAAS,CAAC;AAC5D,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAqD;AAC7D,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,4BAAwB,IAAI,MAAM;AAAA,MAChC,SAAS;AAAA,QACP,SAAS;AAAA,QACT,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,MACd;AAAA,MACA,kBAAkB;AAAA,MAClB,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AAGtF,MAAM,YAAY;AAyDlB,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AAET,eAAW,KAAK,CAAC,cAAc,cAAc,gBAAgB,eAAe,YAAY,UAAU,SAAS,OAAO,UAAU,GAAG;AAC7H,QAAE,aAAa,OAAO,CAAC;AAAA,IACzB;AACA,WAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,OAAO,EAAE;AAAA,EAC/C,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;AAEA,SAAS,aAAa,MAAc,SAA2B;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC;AAC/D,eAAW,KAAK,SAAS;AACvB,YAAM,OAAQ,EAAU,aAAa,MAAM,KAAK;AAChD,UAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,EAAG;AAC9H,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,MAAM,IAAI,EAAE;AAErC,YAAI,IAAI,IAAI,QAAQ,EAAE,WAAW,KAAK,OAAQ;AAC9C,cAAM,IAAI,aAAa,QAAQ;AAC/B,YAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAAE,eAAK,IAAI,CAAC;AAAG,gBAAM,KAAK,QAAQ;AAAA,QAAG;AAAA,MACzD,QAAQ;AAAA,MAAa;AAAA,IACvB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,SAAS,gBAAgB,MAAc,KAAyD;AAC9F,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,yBAAyB,CAAC,EAAG,CAAC,IAAY,OAAO;AACxG,UAAM,SAAS,IAAI,YAAY,UAAiB,EAAE,eAAe,GAAG,CAAC;AACrE,UAAM,UAAU,OAAO,MAAM;AAC7B,QAAI,WAAW,QAAQ,eAAe,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,IAAI;AACnF,YAAM,KAAK,SAAS,SAAS,QAAQ,WAAW,IAAI;AACpD,aAAO,EAAE,UAAU,IAAI,OAAO,QAAQ,SAAS,KAAK;AAAA,IACtD;AAAA,EACF,QAAQ;AAAA,EAAqB;AAE7B,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,sCAAsC,CAAC,EAAG,CAAC,IAAY,OAAO;AACrH,UAAM,WAAY,SAAiB,MAAM,aAAa;AACtD,UAAM,KAAK,SAAS,SAAS,QAAQ;AACrC,UAAM,UAAW,SAAiB,cAAc,OAAO;AACvD,WAAO,EAAE,UAAU,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,KAAK;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,UAAU,IAAI,OAAO,KAAK;AAAA,EACrC;AACF;AAYA,eAAe,iBACb,KACA,OACA,MACA,aAAa,GACS;AACtB,QAAM,KAAK,YAAY,IAAI;AAE3B,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,WAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,EAAE;AAAA,EAC/G;AAEA,QAAM,oBAAoB,MAAM,qBAAqB,GAAG;AAExD,QAAM,UAAkC;AAAA,IACtC,cAAc,KAAK;AAAA,IACnB,UAAU;AAAA,IACV,mBAAmB;AAAA,IACnB,mBAAmB;AAAA;AAAA,IAEnB,aAAa;AAAA,IACb,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,6BAA6B;AAAA,IAC7B,GAAG;AAAA,IACH,GAAI,KAAK,gBAAgB,CAAC;AAAA,EAC5B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,MAAM,yBAAyB;AAClD,eAAW,MAAO,MAAc,KAAK;AAAA,MACnC;AAAA,MACA,QAAQ,YAAY,QAAQ,KAAK,SAAS;AAAA,MAC1C,UAAU;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH,SAAS,GAAQ;AACf,UAAM,IAAI,MAAM,kCAAkC,GAAG,KAAK,EAAE,OAAO,EAAE;AAAA,EACvE;AAEA,QAAM,aAAa,YAAY,IAAI,IAAI;AAGvC,MAAI,SAAS,WAAW,KAAK;AAC3B,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV,YAAM,QAAQ,GAAG;AACjB,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,WAAW;AAAA,IAC5G;AAAA,EACF;AAGA,MAAI,SAAS,WAAW,OAAO,SAAS,WAAW,KAAK;AACtD,QAAI,cAAc,EAAG,OAAM,IAAI,MAAM,+CAA+C,GAAG,EAAE;AACzF,UAAM,aAAa,SAAS,SAAS,QAAQ,IAAI,aAAa,KAAK,GAAG,KAAK;AAC3E,UAAM,YAAY,aAAa,IAAI,aAAa,MAAO,KAAK,IAAI,KAAQ,MAAQ,KAAK,IAAI,GAAG,UAAU,CAAC;AACvG,UAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAQ,IAAI,iBAAiB,SAAS,MAAM,OAAO,GAAG,uBAAkB,KAAK,MAAM,YAAY,MAAM,CAAC,aAAa,aAAa,CAAC,KAAK;AACtI,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,YAAY,MAAM,CAAC;AACxD,WAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,EAC1D;AAGA,MAAI,SAAS,UAAU,KAAK;AAC1B,QAAI,aAAa,GAAG;AAClB,YAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,OAAS,aAAa,EAAE,CAAC;AAC9D,aAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,IAC1D;AACA,QAAI,MAAM,aAAa,GAAG,GAAG;AAC3B,YAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,YAAY,IAAI,IAAI,GAAG;AAAA,IACpI;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAM,OAAO,MAAM,SAAS,KAAK;AACjC,QAAM,OAAO,SAAS,QAAQ,IAAI,MAAM,KAAK;AAC7C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAC9D,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAG9D,MAAI,SAAS,MAAM,YAAY,SAAS,MAAM,GAAG;AAC/C,UAAM,IAAI,KAAK,EAAE,KAAK,MAAM,cAAc,cAAc,MAAM,aAAa,YAAY,SAAS,OAAO,CAAC;AAAA,EAC1G;AAEA,SAAO,EAAE,MAAM,YAAY,SAAS,QAAQ,aAAa,WAAW,OAAO,YAAY,MAAM,aAAa;AAC5G;AAEA,eAAsB,UAAU,MAA+C;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,WAAW,KAAK,YAAY,QAAQ,IAAI,qBAAqB;AACnE,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,QAAQ,IAAI,cAAc,QAAQ;AACxC,QAAM,SAAS,KAAK,qBAAqB,QAAQ,IAAI,YAAY,IAAI;AACrE,QAAM,cAAc,KAAK,aAAa,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,OAAO,KAAK,WAAY,GAAG;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAM;AAAA,EAAE,GAAG,IAAI;AAE7H,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,QAAkB,CAAC,KAAK,MAAM;AACpC,QAAM,SAAS,oBAAI,IAAY,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;AAC1D,QAAM,UAAwB,CAAC;AAC/B,MAAI,cAAc;AAClB,MAAI,aAAa;AACjB,MAAI,mBAAmB;AAEvB,QAAM,aAAa,OAAO,SAAkC;AAC1D,UAAM,QAAQ,IAAI,KAAK,IAAI,OAAO,QAAQ;AACxC,YAAM,WAAW,aAAa,GAAG;AACjC,UAAI,QAAQ,IAAI,QAAQ,EAAG;AAC3B,cAAQ,IAAI,QAAQ;AAGpB,UAAI,UAAU,CAAE,MAAM,OAAO,UAAU,GAAG,EAAE,MAAM,MAAM,IAAI,GAAI;AAC9D,gBAAQ,IAAI,qCAAqC,GAAG,EAAE;AACtD;AAAA,MACF;AAGA,UAAI;AACF,cAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,YAAI,UAAU,KAAK,QAAQ,EAAG;AAAA,MAChC,QAAQ;AAAA,MAAa;AAGrB,UAAI,aAAa;AACf,YAAI;AACF,gBAAM,EAAE,UAAU,OAAO,IAAI,IAAI,IAAI,GAAG;AACxC,cAAI,CAAC,YAAY,KAAK,WAAW,MAAM,GAAG;AACxC,oBAAQ,IAAI,+BAA+B,GAAG,EAAE;AAChD;AAAA,UACF;AAAA,QACF,QAAQ;AAAA,QAAa;AAAA,MACvB;AAEA,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,iBAAiB,KAAK,OAAO,EAAE,WAAW,WAAW,cAAc,KAAK,QAAQ,CAAC;AACrG,YAAI,YAAY,eAAe,OAAO,YAAY,eAAe,IAAK;AAAA,MACxE,SAAS,KAAU;AACjB,gBAAQ,MAAM,gCAAgC,GAAG,KAAK,IAAI,OAAO,EAAE;AACnE;AACA;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,YAAY,SAAS,MAAM,EAAG;AAC/C,UAAI,YAAY,cAAc,IAAK;AACnC,UAAI,YAAY,UAAW;AAE3B,YAAM,EAAE,UAAU,MAAM,IAAI,gBAAgB,YAAY,MAAM,GAAG;AACjE,YAAM,WAAW,gBAAgB,YAAY,MAAM,GAAG;AACtD,YAAM,QAAQ,aAAa,YAAY,MAAM,GAAG;AAChD,YAAM,SAAS,cAAc,YAAY,IAAI;AAC7C,YAAM,SAAS,cAAc,YAAY,MAAM,GAAG;AAElD,YAAM,SAAqB;AAAA,QACzB;AAAA,QACA,OAAO,SAAS,SAAS;AAAA,QACzB;AAAA,QACA,MAAM,KAAK,iBAAiB,YAAY,OAAO;AAAA,QAC/C,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,SAAS,SAAS;AAAA,UAClB,eAAe,SAAS;AAAA,UACxB,SAAS,SAAS;AAAA,UAClB,WAAW,SAAS;AAAA,UACpB,aAAa,SAAS;AAAA,UACtB,MAAM,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE;AAAA,QACpD,YAAY,YAAY;AAAA,QACxB,WAAW,YAAY;AAAA,QACvB,iBAAiB,mBAAmB,UAAU,GAAG,MAAM;AAAA,QACvD,YAAY,YAAY;AAAA,MAC1B;AACA,cAAQ,KAAK,MAAM;AAGnB,yBAAmB,QAAQ,KAAK,YAAY,MAAM,YAAY,UAAU;AAExE,WAAK,aAAa,EAAE,KAAK,cAAc,QAAQ,QAAQ,OAAO,YAAY,QAAQ,YAAY,UAAU,CAAC;AAGzG,iBAAW,QAAQ,OAAO;AACxB,cAAM,IAAI,aAAa,IAAI;AAC3B,YAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,QAAQ,SAAS,MAAM,SAAS,aAAa,GAAG;AACvF,iBAAO,IAAI,CAAC;AACZ,cAAI,KAAK,sBAAsB,MAAO,OAAM,QAAQ,IAAI;AAAA,cACnD,OAAM,KAAK,IAAI;AAAA,QACtB;AAAA,MACF;AAAA,IACF,CAAC,CAAC;AAAA,EACJ;AAEA,QAAM,cAAc,KAAK,eAAe;AAExC,SAAO,MAAM,SAAS,KAAK,QAAQ,SAAS,YAAY;AACtD,UAAM,QAAQ,MAAM,OAAO,GAAG,WAAW;AACzC,UAAM,WAAW,KAAK;AAAA,EACxB;AAGA,MAAI;AACJ,MAAI;AAEJ,QAAM,UAAU,KAAK,iBAAiB,CAAC;AACvC,MAAI,QAAQ,SAAS,WAAW,GAAG;AACjC,gBAAY,QAAQ,IAAI,OAAK,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,UAAU,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACpI;AACA,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS;AACf,UAAM,OAAO,QAAQ;AAAA,MAAI,OACvB,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,SAAS,eAAe,IAAI,EAAE,SAAS,WAAW,IAAI,EAAE,SAAS,YAAY,KAAK,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,OAAK,IAAI,OAAO,CAAC,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG;AAAA,IAC/N;AACA,UAAM,CAAC,QAAQ,GAAG,IAAI,EAAE,KAAK,IAAI;AAAA,EACnC;AAEA,QAAM,aAAa,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,OAAO,QAAQ;AAAA,MACf,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC,cAAc,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
+  "sourcesContent": ["/**\n * HTTP-only crawl engine \u2014 10-50x faster than browser for SSR/static sites.\n *\n * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.\n * No browser overhead. Falls back to browser engine on JS-heavy sites.\n *\n * Features:\n *   - HTTP/2 native (Node 20 built-in fetch)\n *   - ETag / Last-Modified response caching\n *   - 429 / 503 exponential backoff with jitter\n *   - BFS / DFS traversal\n *   - URL regex filtering\n *   - Robots.txt compliance\n *   - Link discovery (href, action, src)\n *   - Markdown extraction (Readability + Turndown)\n *   - OG / schema.org / meta extraction\n *   - Table \u2192 JSON extraction\n *   - JSON Lines, CSV, raw HTML output formats\n *   - Streaming progress via onProgress callback\n */\n\nimport { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { parseHTML } from 'linkedom';\nimport { RobotsCache } from './robots-txt.js';\nimport { ResponseCache } from './response-cache.js';\nimport { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';\nimport { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';\nimport { renderingPredictor } from './engine-router.js';\n\n// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback\nlet chromeFetchDispatcher: any;\nasync function getChromeFetchDispatcher() {\n  if (!chromeFetchDispatcher) {\n    try {\n      // impit provides native Chrome BoringSSL TLS fingerprint \u2014 true JA3/JA4 match\n      // @ts-ignore \u2014 impit is an optional dependency\n      const impitMod = await import('impit');\n      const ImpitClass = impitMod.Impit ?? impitMod.default;\n      if (ImpitClass) {\n        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });\n        return chromeFetchDispatcher;\n      }\n    } catch { /* impit not installed \u2014 fall through to undici */ }\n    const { Agent } = await import('undici');\n    chromeFetchDispatcher = new Agent({\n      connect: {\n        ciphers: CHROME_CIPHERS,\n        ecdhCurve: CHROME_CURVES,\n        minVersion: 'TLSv1.2' as any,\n        maxVersion: 'TLSv1.3' as any,\n      },\n      keepAliveTimeout: 10_000,\n      keepAliveMaxTimeout: 30_000,\n    });\n  }\n  return chromeFetchDispatcher;\n}\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\n\n// Pages unlikely to have testable UI \u2014 skip from HTTP crawl queue\nconst SKIP_PATH = /\\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;\n\nexport type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';\n\nexport interface HttpScreen {\n  url: string;\n  title: string | null;\n  markdown: string;\n  html?: string;\n  metadata: {\n    description: string | null;\n    ogTitle: string | null;\n    ogDescription: string | null;\n    ogImage: string | null;\n    canonical: string | null;\n    schemaTypes: string[];\n    lang: string | null;\n  };\n  tables: Array<Record<string, string>[]>;\n  links: string[];\n  images: Array<{ src: string; alt: string }>;\n  statusCode: number;\n  fromCache: boolean;\n  requiresBrowser?: boolean;\n  durationMs: number;\n}\n\nexport interface HttpCrawlOpts {\n  appUrl: string;\n  maxScreens?: number;            // default 100\n  urlFilter?: string;             // regex pattern \u2014 only crawl matching URLs\n  traversalStrategy?: 'BFS' | 'DFS'; // default BFS\n  outputFormats?: HttpOutputFormat[];\n  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'\n  userAgent?: string;\n  headers?: Record<string, string>;\n  respectRobotsTxt?: boolean;     // default true\n  includeRawHtml?: boolean;\n  timeoutMs?: number;             // per-request timeout, default 15000\n  concurrency?: number;           // default 5 parallel fetches\n  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;\n}\n\nexport interface HttpCrawlResult {\n  screens: HttpScreen[];\n  stats: {\n    total: number;\n    cached: number;\n    errors: number;\n    rateLimited: number;\n    durationMs: number;\n    cacheHitRate: number;\n  };\n  jsonLines?: string;\n  csv?: string;\n}\n\nfunction canonicalize(url: string): string {\n  try {\n    const u = new URL(url);\n    u.hash = '';\n    // Strip tracking params\n    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {\n      u.searchParams.delete(p);\n    }\n    return u.href.toLowerCase().replace(/\\/$/, '');\n  } catch {\n    return url.toLowerCase();\n  }\n}\n\nfunction extractLinks(html: string, baseUrl: string): string[] {\n  const links: string[] = [];\n  const seen = new Set<string>();\n  try {\n    const base = new URL(baseUrl);\n    const { document } = parseHTML(html);\n    const anchors = Array.from(document.querySelectorAll('a[href]'));\n    for (const a of anchors) {\n      const href = (a as any).getAttribute('href') ?? '';\n      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;\n      try {\n        const resolved = new URL(href, base).href;\n        // Same origin only\n        if (new URL(resolved).origin !== base.origin) continue;\n        const c = canonicalize(resolved);\n        if (!seen.has(c)) { seen.add(c); links.push(resolved); }\n      } catch { /* skip */ }\n    }\n  } catch { /* non-fatal */ }\n  return links;\n}\n\nfunction extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {\n  try {\n    const { document } = parseHTML(html);\n    // Remove script/style tags before Readability\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();\n    const reader = new Readability(document as any, { charThreshold: 50 });\n    const article = reader.parse();\n    if (article && article.textContent && article.textContent.split(/\\s+/).length >= 30) {\n      const md = turndown.turndown(article.content ?? html);\n      return { markdown: md, title: article.title ?? null };\n    }\n  } catch { /* fall through */ }\n  // Full-page fallback\n  try {\n    const { document } = parseHTML(html);\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();\n    const bodyHtml = (document as any).body?.innerHTML ?? html;\n    const md = turndown.turndown(bodyHtml);\n    const titleEl = (document as any).querySelector('title');\n    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };\n  } catch {\n    return { markdown: '', title: null };\n  }\n}\n\ninterface FetchResult {\n  body: string;\n  statusCode: number;\n  contentType: string;\n  fromCache: boolean;\n  durationMs: number;\n  etag?: string;\n  lastModified?: string;\n}\n\nasync function fetchWithBackoff(\n  url: string,\n  cache: ResponseCache,\n  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },\n  retryCount = 0\n): Promise<FetchResult> {\n  const t0 = performance.now();\n\n  if (cache.isFresh(url)) {\n    const cached = cache.get(url)!;\n    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };\n  }\n\n  const validationHeaders = cache.getValidationHeaders(url);\n\n  const headers: Record<string, string> = {\n    'User-Agent': opts.userAgent,\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n    'Accept-Language': 'en-US,en;q=0.9',\n    'Accept-Encoding': 'gzip, deflate, br, zstd',\n    // Chrome-like security headers reduce bot-detection fingerprint\n    'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n    'sec-ch-ua-mobile': '?0',\n    'sec-ch-ua-platform': '\"Windows\"',\n    'Sec-Fetch-Dest': 'document',\n    'Sec-Fetch-Mode': 'navigate',\n    'Sec-Fetch-Site': 'none',\n    'Sec-Fetch-User': '?1',\n    'Upgrade-Insecure-Requests': '1',\n    ...validationHeaders,\n    ...(opts.extraHeaders ?? {}),\n  };\n\n  let response: Response;\n  try {\n    const dispatcher = await getChromeFetchDispatcher();\n    response = await (fetch as any)(url, {\n      headers,\n      signal: AbortSignal.timeout(opts.timeoutMs),\n      redirect: 'follow',\n      dispatcher,\n    });\n  } catch (e: any) {\n    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);\n  }\n\n  const durationMs = performance.now() - t0;\n\n  // 304 Not Modified \u2014 use cached body\n  if (response.status === 304) {\n    const cached = cache.get(url);\n    if (cached) {\n      cache.markHit(url);\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };\n    }\n  }\n\n  // 429 / 503 \u2014 exponential backoff with jitter\n  if (response.status === 429 || response.status === 503) {\n    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);\n    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;\n    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));\n    const jitter = Math.random() * 1000;\n    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);\n    await new Promise(r => setTimeout(r, baseDelay + jitter));\n    return fetchWithBackoff(url, cache, opts, retryCount + 1);\n  }\n\n  // 5xx \u2014 retry up to 3 times\n  if (response.status >= 500) {\n    if (retryCount < 3) {\n      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));\n      return fetchWithBackoff(url, cache, opts, retryCount + 1);\n    }\n    if (cache.staleIfError(url)) {\n      const cached = cache.get(url)!;\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };\n    }\n  }\n\n  const contentType = response.headers.get('content-type') ?? '';\n  const body = await response.text();\n  const etag = response.headers.get('etag') ?? undefined;\n  const lastModified = response.headers.get('last-modified') ?? undefined;\n  const cacheControl = response.headers.get('cache-control') ?? undefined;\n\n  // Cache successful HTML responses\n  if (response.ok && contentType.includes('html')) {\n    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });\n  }\n\n  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };\n}\n\nexport async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {\n  const startTime = performance.now();\n  const maxScreens = opts.maxScreens ?? 100;\n  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';\n  const timeoutMs = opts.timeoutMs ?? 15_000;\n  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';\n  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;\n  const cache = new ResponseCache(cacheDir);\n  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;\n  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;\n\n  const visited = new Set<string>();\n  const queue: string[] = [opts.appUrl];\n  const queued = new Set<string>([canonicalize(opts.appUrl)]);\n  const screens: HttpScreen[] = [];\n  let cachedCount = 0;\n  let errorCount = 0;\n  let rateLimitedCount = 0;\n\n  const batchFetch = async (urls: string[]): Promise<void> => {\n    await Promise.all(urls.map(async (url) => {\n      const canonUrl = canonicalize(url);\n      if (visited.has(canonUrl)) return;\n      visited.add(canonUrl);\n\n      // Robots.txt check\n      if (robots && !(await robots.isAllowed(url).catch(() => true))) {\n        console.log(`[http-engine] robots.txt blocked: ${url}`);\n        return;\n      }\n\n      // Skip content/legal pages\n      try {\n        const { pathname } = new URL(url);\n        if (SKIP_PATH.test(pathname)) return;\n      } catch { /* skip */ }\n\n      // URL filter\n      if (urlFilterRe) {\n        try {\n          const { pathname, search } = new URL(url);\n          if (!urlFilterRe.test(pathname + search)) {\n            console.log(`[http-engine] filtered out: ${url}`);\n            return;\n          }\n        } catch { /* skip */ }\n      }\n\n      let fetchResult: FetchResult;\n      try {\n        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });\n        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;\n      } catch (err: any) {\n        console.error(`[http-engine] error fetching ${url}: ${err.message}`);\n        errorCount++;\n        return;\n      }\n\n      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML\n      if (fetchResult.statusCode >= 400) return;\n      if (fetchResult.fromCache) cachedCount++;\n\n      const { markdown, title } = extractMarkdown(fetchResult.body, url);\n      const fullMeta = extractMetadata(fetchResult.body, url);\n      const links = extractLinks(fetchResult.body, url);\n      const tables = extractTables(fetchResult.body);\n      const images = extractImages(fetchResult.body, url);\n\n      const screen: HttpScreen = {\n        url,\n        title: title ?? fullMeta.title,\n        markdown,\n        html: opts.includeRawHtml ? fetchResult.body : undefined,\n        metadata: {\n          description: fullMeta.description,\n          ogTitle: fullMeta.ogTitle,\n          ogDescription: fullMeta.ogDescription,\n          ogImage: fullMeta.ogImage,\n          canonical: fullMeta.canonical,\n          schemaTypes: fullMeta.schemaTypes,\n          lang: fullMeta.lang,\n        },\n        tables,\n        links,\n        images: images.map(i => ({ src: i.src, alt: i.alt })),\n        statusCode: fetchResult.statusCode,\n        fromCache: fetchResult.fromCache,\n        requiresBrowser: renderingPredictor.getCached(url) === 'browser',\n        durationMs: fetchResult.durationMs,\n      };\n      screens.push(screen);\n\n      // Record rendering prediction: SPA signals \u2192 browser needed; content-rich \u2192 http engine works\n      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);\n\n      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });\n\n      // Enqueue discovered links\n      for (const link of links) {\n        const c = canonicalize(link);\n        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {\n          queued.add(c);\n          if (opts.traversalStrategy === 'DFS') queue.unshift(link);\n          else queue.push(link);\n        }\n      }\n    }));\n  };\n\n  const concurrency = opts.concurrency ?? 5;\n\n  while (queue.length > 0 && screens.length < maxScreens) {\n    const batch = queue.splice(0, concurrency);\n    await batchFetch(batch);\n  }\n\n  // Output formats\n  let jsonLines: string | undefined;\n  let csv: string | undefined;\n\n  const formats = opts.outputFormats ?? [];\n  if (formats.includes('jsonlines')) {\n    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\\n');\n  }\n  if (formats.includes('csv')) {\n    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';\n    const rows = screens.map(s =>\n      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `\"${String(v).replace(/\"/g, '\"\"')}\"`).join(',')\n    );\n    csv = [header, ...rows].join('\\n');\n  }\n\n  const cacheStats = cache.stats();\n  return {\n    screens,\n    stats: {\n      total: screens.length,\n      cached: cachedCount,\n      errors: errorCount,\n      rateLimited: rateLimitedCount,\n      durationMs: performance.now() - startTime,\n      cacheHitRate: cacheStats.hitRate,\n    },\n    jsonLines,\n    csv,\n  };\n}\n"],
+  "mappings": "AAwBA,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB,eAAe,qBAAqB;AAC9D,SAAS,gBAAgB,qBAAqB;AAC9C,SAAS,0BAA0B;AAGnC,IAAI;AACJ,eAAe,2BAA2B;AACxC,MAAI,CAAC,uBAAuB;AAC1B,QAAI;AAGF,YAAM,WAAW,MAAM,OAAO,OAAO;AACrC,YAAM,aAAa,SAAS,SAAS,SAAS;AAC9C,UAAI,YAAY;AACd,gCAAwB,IAAI,WAAW,EAAE,SAAS,SAAS,CAAC;AAC5D,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAqD;AAC7D,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,4BAAwB,IAAI,MAAM;AAAA,MAChC,SAAS;AAAA,QACP,SAAS;AAAA,QACT,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,MACd;AAAA,MACA,kBAAkB;AAAA,MAClB,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AAGtF,MAAM,YAAY;AAyDlB,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AAET,eAAW,KAAK,CAAC,cAAc,cAAc,gBAAgB,eAAe,YAAY,UAAU,SAAS,OAAO,UAAU,GAAG;AAC7H,QAAE,aAAa,OAAO,CAAC;AAAA,IACzB;AACA,WAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,OAAO,EAAE;AAAA,EAC/C,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;AAEA,SAAS,aAAa,MAAc,SAA2B;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC;AAC/D,eAAW,KAAK,SAAS;AACvB,YAAM,OAAQ,EAAU,aAAa,MAAM,KAAK;AAChD,UAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,EAAG;AAC9H,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,MAAM,IAAI,EAAE;AAErC,YAAI,IAAI,IAAI,QAAQ,EAAE,WAAW,KAAK,OAAQ;AAC9C,cAAM,IAAI,aAAa,QAAQ;AAC/B,YAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAAE,eAAK,IAAI,CAAC;AAAG,gBAAM,KAAK,QAAQ;AAAA,QAAG;AAAA,MACzD,QAAQ;AAAA,MAAa;AAAA,IACvB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,SAAS,gBAAgB,MAAc,KAAyD;AAC9F,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,yBAAyB,CAAC,EAAG,CAAC,IAAY,OAAO;AACxG,UAAM,SAAS,IAAI,YAAY,UAAiB,EAAE,eAAe,GAAG,CAAC;AACrE,UAAM,UAAU,OAAO,MAAM;AAC7B,QAAI,WAAW,QAAQ,eAAe,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,IAAI;AACnF,YAAM,KAAK,SAAS,SAAS,QAAQ,WAAW,IAAI;AACpD,aAAO,EAAE,UAAU,IAAI,OAAO,QAAQ,SAAS,KAAK;AAAA,IACtD;AAAA,EACF,QAAQ;AAAA,EAAqB;AAE7B,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,sCAAsC,CAAC,EAAG,CAAC,IAAY,OAAO;AACrH,UAAM,WAAY,SAAiB,MAAM,aAAa;AACtD,UAAM,KAAK,SAAS,SAAS,QAAQ;AACrC,UAAM,UAAW,SAAiB,cAAc,OAAO;AACvD,WAAO,EAAE,UAAU,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,KAAK;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,UAAU,IAAI,OAAO,KAAK;AAAA,EACrC;AACF;AAYA,eAAe,iBACb,KACA,OACA,MACA,aAAa,GACS;AACtB,QAAM,KAAK,YAAY,IAAI;AAE3B,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,WAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,EAAE;AAAA,EAC/G;AAEA,QAAM,oBAAoB,MAAM,qBAAqB,GAAG;AAExD,QAAM,UAAkC;AAAA,IACtC,cAAc,KAAK;AAAA,IACnB,UAAU;AAAA,IACV,mBAAmB;AAAA,IACnB,mBAAmB;AAAA;AAAA,IAEnB,aAAa;AAAA,IACb,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,6BAA6B;AAAA,IAC7B,GAAG;AAAA,IACH,GAAI,KAAK,gBAAgB,CAAC;AAAA,EAC5B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,MAAM,yBAAyB;AAClD,eAAW,MAAO,MAAc,KAAK;AAAA,MACnC;AAAA,MACA,QAAQ,YAAY,QAAQ,KAAK,SAAS;AAAA,MAC1C,UAAU;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH,SAAS,GAAQ;AACf,UAAM,IAAI,MAAM,kCAAkC,GAAG,KAAK,EAAE,OAAO,EAAE;AAAA,EACvE;AAEA,QAAM,aAAa,YAAY,IAAI,IAAI;AAGvC,MAAI,SAAS,WAAW,KAAK;AAC3B,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV,YAAM,QAAQ,GAAG;AACjB,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,WAAW;AAAA,IAC5G;AAAA,EACF;AAGA,MAAI,SAAS,WAAW,OAAO,SAAS,WAAW,KAAK;AACtD,QAAI,cAAc,EAAG,OAAM,IAAI,MAAM,+CAA+C,GAAG,EAAE;AACzF,UAAM,aAAa,SAAS,SAAS,QAAQ,IAAI,aAAa,KAAK,GAAG,KAAK;AAC3E,UAAM,YAAY,aAAa,IAAI,aAAa,MAAO,KAAK,IAAI,KAAQ,MAAQ,KAAK,IAAI,GAAG,UAAU,CAAC;AACvG,UAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAQ,IAAI,iBAAiB,SAAS,MAAM,OAAO,GAAG,uBAAkB,KAAK,MAAM,YAAY,MAAM,CAAC,aAAa,aAAa,CAAC,KAAK;AACtI,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,YAAY,MAAM,CAAC;AACxD,WAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,EAC1D;AAGA,MAAI,SAAS,UAAU,KAAK;AAC1B,QAAI,aAAa,GAAG;AAClB,YAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,OAAS,aAAa,EAAE,CAAC;AAC9D,aAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,IAC1D;AACA,QAAI,MAAM,aAAa,GAAG,GAAG;AAC3B,YAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,YAAY,IAAI,IAAI,GAAG;AAAA,IACpI;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAM,OAAO,MAAM,SAAS,KAAK;AACjC,QAAM,OAAO,SAAS,QAAQ,IAAI,MAAM,KAAK;AAC7C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAC9D,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAG9D,MAAI,SAAS,MAAM,YAAY,SAAS,MAAM,GAAG;AAC/C,UAAM,IAAI,KAAK,EAAE,KAAK,MAAM,cAAc,cAAc,MAAM,aAAa,YAAY,SAAS,OAAO,CAAC;AAAA,EAC1G;AAEA,SAAO,EAAE,MAAM,YAAY,SAAS,QAAQ,aAAa,WAAW,OAAO,YAAY,MAAM,aAAa;AAC5G;AAEA,eAAsB,UAAU,MAA+C;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,WAAW,KAAK,YAAY,QAAQ,IAAI,qBAAqB;AACnE,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,qBAAqB,QAAQ,IAAI,sBAAsB;AAC7D,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,QAAQ,IAAI,cAAc,QAAQ;AACxC,QAAM,SAAS,KAAK,qBAAqB,QAAQ,IAAI,YAAY,IAAI;AACrE,QAAM,cAAc,KAAK,aAAa,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,OAAO,KAAK,WAAY,GAAG;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAM;AAAA,EAAE,GAAG,IAAI;AAE7H,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,QAAkB,CAAC,KAAK,MAAM;AACpC,QAAM,SAAS,oBAAI,IAAY,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;AAC1D,QAAM,UAAwB,CAAC;AAC/B,MAAI,cAAc;AAClB,MAAI,aAAa;AACjB,MAAI,mBAAmB;AAEvB,QAAM,aAAa,OAAO,SAAkC;AAC1D,UAAM,QAAQ,IAAI,KAAK,IAAI,OAAO,QAAQ;AACxC,YAAM,WAAW,aAAa,GAAG;AACjC,UAAI,QAAQ,IAAI,QAAQ,EAAG;AAC3B,cAAQ,IAAI,QAAQ;AAGpB,UAAI,UAAU,CAAE,MAAM,OAAO,UAAU,GAAG,EAAE,MAAM,MAAM,IAAI,GAAI;AAC9D,gBAAQ,IAAI,qCAAqC,GAAG,EAAE;AACtD;AAAA,MACF;AAGA,UAAI;AACF,cAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,YAAI,UAAU,KAAK,QAAQ,EAAG;AAAA,MAChC,QAAQ;AAAA,MAAa;AAGrB,UAAI,aAAa;AACf,YAAI;AACF,gBAAM,EAAE,UAAU,OAAO,IAAI,IAAI,IAAI,GAAG;AACxC,cAAI,CAAC,YAAY,KAAK,WAAW,MAAM,GAAG;AACxC,oBAAQ,IAAI,+BAA+B,GAAG,EAAE;AAChD;AAAA,UACF;AAAA,QACF,QAAQ;AAAA,QAAa;AAAA,MACvB;AAEA,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,iBAAiB,KAAK,OAAO,EAAE,WAAW,WAAW,cAAc,KAAK,QAAQ,CAAC;AACrG,YAAI,YAAY,eAAe,OAAO,YAAY,eAAe,IAAK;AAAA,MACxE,SAAS,KAAU;AACjB,gBAAQ,MAAM,gCAAgC,GAAG,KAAK,IAAI,OAAO,EAAE;AACnE;AACA;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,YAAY,SAAS,MAAM,EAAG;AAC/C,UAAI,YAAY,cAAc,IAAK;AACnC,UAAI,YAAY,UAAW;AAE3B,YAAM,EAAE,UAAU,MAAM,IAAI,gBAAgB,YAAY,MAAM,GAAG;AACjE,YAAM,WAAW,gBAAgB,YAAY,MAAM,GAAG;AACtD,YAAM,QAAQ,aAAa,YAAY,MAAM,GAAG;AAChD,YAAM,SAAS,cAAc,YAAY,IAAI;AAC7C,YAAM,SAAS,cAAc,YAAY,MAAM,GAAG;AAElD,YAAM,SAAqB;AAAA,QACzB;AAAA,QACA,OAAO,SAAS,SAAS;AAAA,QACzB;AAAA,QACA,MAAM,KAAK,iBAAiB,YAAY,OAAO;AAAA,QAC/C,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,SAAS,SAAS;AAAA,UAClB,eAAe,SAAS;AAAA,UACxB,SAAS,SAAS;AAAA,UAClB,WAAW,SAAS;AAAA,UACpB,aAAa,SAAS;AAAA,UACtB,MAAM,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE;AAAA,QACpD,YAAY,YAAY;AAAA,QACxB,WAAW,YAAY;AAAA,QACvB,iBAAiB,mBAAmB,UAAU,GAAG,MAAM;AAAA,QACvD,YAAY,YAAY;AAAA,MAC1B;AACA,cAAQ,KAAK,MAAM;AAGnB,yBAAmB,QAAQ,KAAK,YAAY,MAAM,YAAY,UAAU;AAExE,WAAK,aAAa,EAAE,KAAK,cAAc,QAAQ,QAAQ,OAAO,YAAY,QAAQ,YAAY,UAAU,CAAC;AAGzG,iBAAW,QAAQ,OAAO;AACxB,cAAM,IAAI,aAAa,IAAI;AAC3B,YAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,QAAQ,SAAS,MAAM,SAAS,aAAa,GAAG;AACvF,iBAAO,IAAI,CAAC;AACZ,cAAI,KAAK,sBAAsB,MAAO,OAAM,QAAQ,IAAI;AAAA,cACnD,OAAM,KAAK,IAAI;AAAA,QACtB;AAAA,MACF;AAAA,IACF,CAAC,CAAC;AAAA,EACJ;AAEA,QAAM,cAAc,KAAK,eAAe;AAExC,SAAO,MAAM,SAAS,KAAK,QAAQ,SAAS,YAAY;AACtD,UAAM,QAAQ,MAAM,OAAO,GAAG,WAAW;AACzC,UAAM,WAAW,KAAK;AAAA,EACxB;AAGA,MAAI;AACJ,MAAI;AAEJ,QAAM,UAAU,KAAK,iBAAiB,CAAC;AACvC,MAAI,QAAQ,SAAS,WAAW,GAAG;AACjC,gBAAY,QAAQ,IAAI,OAAK,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,UAAU,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACpI;AACA,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS;AACf,UAAM,OAAO,QAAQ;AAAA,MAAI,OACvB,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,SAAS,eAAe,IAAI,EAAE,SAAS,WAAW,IAAI,EAAE,SAAS,YAAY,KAAK,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,OAAK,IAAI,OAAO,CAAC,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG;AAAA,IAC/N;AACA,UAAM,CAAC,QAAQ,GAAG,IAAI,EAAE,KAAK,IAAI;AAAA,EACnC;AAEA,QAAM,aAAa,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,OAAO,QAAQ;AAAA,MACf,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC,cAAc,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/job-queue.js

@@ -416,11 +416,13 @@ async function processJob(jobId, config, opts) {
       ...config.credentials,
       ...config.credentials.totpSecret ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) } : {}
     } : void 0;
+    const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
     const result = await crawlProject({
       ...config,
       credentials: runtimeCreds,
       abortSignal: controller.signal,
       jobId,
+      storageConfig: storageConfigForCrawl,
       onCheckpoint: async (visitedUrls, pendingQueue) => {
         jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
       }
@@ -1066,7 +1068,8 @@ async function processRunTestSuiteJob(jobId, config) {
       return;
     }
     const aiHealingEnabled = await isFeatureEnabled(tenantId, "AI_SELF_HEALING");
-    const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || "/tmp/zeta-screenshots";
+    const SCREENSHOT_DIR = process.env.SCREENSHOT_DIR ?? "/tmp/zeta-screenshots";
+    const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || SCREENSHOT_DIR;
     const VALID_BROWSERS = ["chromium", "firefox", "webkit"];
     const validBrowser = VALID_BROWSERS.includes(browser) ? browser : "chromium";
     let passed = 0;
@@ -1469,7 +1472,7 @@ async function processMonitorJob(jobId, config, opts) {
       await updateJobProgress(tenantId, jobId, pct, `Checking ${url}...`);
       opts?.onProgress?.(pct);
       try {
-        const crawlResult = await httpCrawl({ appUrl: url, maxScreens: 1, respectRobotsTxt: false, timeoutMs: 2e4 });
+        const crawlResult = await httpCrawl({ appUrl: url, maxScreens: 1, respectRobotsTxt: false, timeoutMs: parseInt(process.env.CRAWL_TIMEOUT_MS ?? "20000", 10) });
         const screen = crawlResult.screens[0];
         if (!screen) {
           results.push({ url, changed: false, reason: "fetch_empty", currentHash: "", previousHash: null });