@detiq/crawler 1.1.13 → 1.1.15

package/dist/http-engine.js

@@ -33,7 +33,19 @@ async function getChromeFetchDispatcher() {
   return chromeFetchDispatcher;
 }
 const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
-const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA ? process.env.CRAWL_SKIP_PATH_EXTRA.split(",").map((p) => p.trim()).filter(Boolean) : [];
+const SKIP_PATH = _skipPathExtra.length > 0 ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join("|")})([-/]|$)`, "i") : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+async function getHttpConfig() {
+  const { getSystemConfig } = await import("@detiq/core");
+  const cfg = await getSystemConfig("crawl_tuning");
+  return {
+    retryLimit: parseInt(String(cfg?.httpRetryLimit ?? "5"), 10),
+    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs ?? "60000"), 10),
+    retry5xx: parseInt(String(cfg?.http5xxRetryLimit ?? "3"), 10),
+    timeoutMs: parseInt(String(cfg?.httpTimeoutMs ?? "15000"), 10),
+    concurrency: parseInt(String(cfg?.httpConcurrency ?? "5"), 10)
+  };
+}
 function canonicalize(url) {
   try {
     const u = new URL(url);
@@ -94,7 +106,8 @@ function extractMarkdown(html, url) {
     return { markdown: "", title: null };
   }
 }
-async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
+async function fetchWithBackoff(url, cache, opts, retryCount = 0, httpCfg) {
+  if (!httpCfg) httpCfg = await getHttpConfig();
   const t0 = performance.now();
   if (cache.isFresh(url)) {
     const cached = cache.get(url);
@@ -107,7 +120,7 @@ async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
     "Accept-Language": "en-US,en;q=0.9",
     "Accept-Encoding": "gzip, deflate, br, zstd",
     // Chrome-like security headers reduce bot-detection fingerprint
-    "sec-ch-ua": '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    "sec-ch-ua": `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? "120"}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? "120"}", "Not-A.Brand";v="99"`,
     "sec-ch-ua-mobile": "?0",
     "sec-ch-ua-platform": '"Windows"',
     "Sec-Fetch-Dest": "document",
@@ -139,18 +152,18 @@ async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
     }
   }
   if (response.status === 429 || response.status === 503) {
-    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
     const retryAfter = parseInt(response.headers.get("retry-after") ?? "0") || 0;
-    const baseDelay = retryAfter > 0 ? retryAfter * 1e3 : Math.min(6e4, 1e3 * Math.pow(2, retryCount));
+    const baseDelay = retryAfter > 0 ? retryAfter * 1e3 : Math.min(httpCfg.backoffCapMs, 1e3 * Math.pow(2, retryCount));
     const jitter = Math.random() * 1e3;
-    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
     await new Promise((r) => setTimeout(r, baseDelay + jitter));
-    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
   }
   if (response.status >= 500) {
-    if (retryCount < 3) {
+    if (retryCount < httpCfg.retry5xx) {
       await new Promise((r) => setTimeout(r, 2e3 * (retryCount + 1)));
-      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
     }
     if (cache.staleIfError(url)) {
       const cached = cache.get(url);
@@ -169,11 +182,11 @@ async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
 }
 async function httpCrawl(opts) {
   const startTime = performance.now();
+  const httpCfg = await getHttpConfig();
   const maxScreens = opts.maxScreens ?? 100;
-  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? "/tmp/zeta-cache";
-  const timeoutMs = opts.timeoutMs ?? 15e3;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? "ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)";
-  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
+  const cacheDir = opts.cacheDir ?? "/tmp/zeta-cache";
+  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
+  const userAgent = opts.userAgent ?? "ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)";
   const cache = new ResponseCache(cacheDir);
   const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
   const urlFilterRe = opts.urlFilter ? (() => {
@@ -216,7 +229,7 @@ async function httpCrawl(opts) {
       }
       let fetchResult;
       try {
-        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
         if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
       } catch (err) {
         console.error(`[http-engine] error fetching ${url}: ${err.message}`);
@@ -266,7 +279,7 @@ async function httpCrawl(opts) {
       }
     }));
   };
-  const concurrency = opts.concurrency ?? 5;
+  const concurrency = opts.concurrency ?? httpCfg.concurrency;
   while (queue.length > 0 && screens.length < maxScreens) {
     const batch = queue.splice(0, concurrency);
     await batchFetch(batch);

package/dist/http-engine.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/http-engine.ts"],
-  "sourcesContent": ["/**\n * HTTP-only crawl engine \u2014 10-50x faster than browser for SSR/static sites.\n *\n * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.\n * No browser overhead. Falls back to browser engine on JS-heavy sites.\n *\n * Features:\n *   - HTTP/2 native (Node 20 built-in fetch)\n *   - ETag / Last-Modified response caching\n *   - 429 / 503 exponential backoff with jitter\n *   - BFS / DFS traversal\n *   - URL regex filtering\n *   - Robots.txt compliance\n *   - Link discovery (href, action, src)\n *   - Markdown extraction (Readability + Turndown)\n *   - OG / schema.org / meta extraction\n *   - Table \u2192 JSON extraction\n *   - JSON Lines, CSV, raw HTML output formats\n *   - Streaming progress via onProgress callback\n */\n\nimport { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { parseHTML } from 'linkedom';\nimport { RobotsCache } from './robots-txt.js';\nimport { ResponseCache } from './response-cache.js';\nimport { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';\nimport { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';\nimport { renderingPredictor } from './engine-router.js';\n\n// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback\nlet chromeFetchDispatcher: any;\nasync function getChromeFetchDispatcher() {\n  if (!chromeFetchDispatcher) {\n    try {\n      // impit provides native Chrome BoringSSL TLS fingerprint \u2014 true JA3/JA4 match\n      // @ts-ignore \u2014 impit is an optional dependency\n      const impitMod = await import('impit');\n      const ImpitClass = impitMod.Impit ?? impitMod.default;\n      if (ImpitClass) {\n        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });\n        return chromeFetchDispatcher;\n      }\n    } catch { /* impit not installed \u2014 fall through to undici */ }\n    const { Agent } = await import('undici');\n    chromeFetchDispatcher = new Agent({\n      connect: {\n        ciphers: CHROME_CIPHERS,\n        ecdhCurve: CHROME_CURVES,\n        minVersion: 'TLSv1.2' as any,\n        maxVersion: 'TLSv1.3' as any,\n      },\n      keepAliveTimeout: 10_000,\n      keepAliveMaxTimeout: 30_000,\n    });\n  }\n  return chromeFetchDispatcher;\n}\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\n\n// Pages unlikely to have testable UI \u2014 skip from HTTP crawl queue\nconst SKIP_PATH = /\\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;\n\nexport type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';\n\nexport interface HttpScreen {\n  url: string;\n  title: string | null;\n  markdown: string;\n  html?: string;\n  metadata: {\n    description: string | null;\n    ogTitle: string | null;\n    ogDescription: string | null;\n    ogImage: string | null;\n    canonical: string | null;\n    schemaTypes: string[];\n    lang: string | null;\n  };\n  tables: Array<Record<string, string>[]>;\n  links: string[];\n  images: Array<{ src: string; alt: string }>;\n  statusCode: number;\n  fromCache: boolean;\n  requiresBrowser?: boolean;\n  durationMs: number;\n}\n\nexport interface HttpCrawlOpts {\n  appUrl: string;\n  maxScreens?: number;            // default 100\n  urlFilter?: string;             // regex pattern \u2014 only crawl matching URLs\n  traversalStrategy?: 'BFS' | 'DFS'; // default BFS\n  outputFormats?: HttpOutputFormat[];\n  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'\n  userAgent?: string;\n  headers?: Record<string, string>;\n  respectRobotsTxt?: boolean;     // default true\n  includeRawHtml?: boolean;\n  timeoutMs?: number;             // per-request timeout, default 15000\n  concurrency?: number;           // default 5 parallel fetches\n  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;\n}\n\nexport interface HttpCrawlResult {\n  screens: HttpScreen[];\n  stats: {\n    total: number;\n    cached: number;\n    errors: number;\n    rateLimited: number;\n    durationMs: number;\n    cacheHitRate: number;\n  };\n  jsonLines?: string;\n  csv?: string;\n}\n\nfunction canonicalize(url: string): string {\n  try {\n    const u = new URL(url);\n    u.hash = '';\n    // Strip tracking params\n    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {\n      u.searchParams.delete(p);\n    }\n    return u.href.toLowerCase().replace(/\\/$/, '');\n  } catch {\n    return url.toLowerCase();\n  }\n}\n\nfunction extractLinks(html: string, baseUrl: string): string[] {\n  const links: string[] = [];\n  const seen = new Set<string>();\n  try {\n    const base = new URL(baseUrl);\n    const { document } = parseHTML(html);\n    const anchors = Array.from(document.querySelectorAll('a[href]'));\n    for (const a of anchors) {\n      const href = (a as any).getAttribute('href') ?? '';\n      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;\n      try {\n        const resolved = new URL(href, base).href;\n        // Same origin only\n        if (new URL(resolved).origin !== base.origin) continue;\n        const c = canonicalize(resolved);\n        if (!seen.has(c)) { seen.add(c); links.push(resolved); }\n      } catch { /* skip */ }\n    }\n  } catch { /* non-fatal */ }\n  return links;\n}\n\nfunction extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {\n  try {\n    const { document } = parseHTML(html);\n    // Remove script/style tags before Readability\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();\n    const reader = new Readability(document as any, { charThreshold: 50 });\n    const article = reader.parse();\n    if (article && article.textContent && article.textContent.split(/\\s+/).length >= 30) {\n      const md = turndown.turndown(article.content ?? html);\n      return { markdown: md, title: article.title ?? null };\n    }\n  } catch { /* fall through */ }\n  // Full-page fallback\n  try {\n    const { document } = parseHTML(html);\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();\n    const bodyHtml = (document as any).body?.innerHTML ?? html;\n    const md = turndown.turndown(bodyHtml);\n    const titleEl = (document as any).querySelector('title');\n    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };\n  } catch {\n    return { markdown: '', title: null };\n  }\n}\n\ninterface FetchResult {\n  body: string;\n  statusCode: number;\n  contentType: string;\n  fromCache: boolean;\n  durationMs: number;\n  etag?: string;\n  lastModified?: string;\n}\n\nasync function fetchWithBackoff(\n  url: string,\n  cache: ResponseCache,\n  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },\n  retryCount = 0\n): Promise<FetchResult> {\n  const t0 = performance.now();\n\n  if (cache.isFresh(url)) {\n    const cached = cache.get(url)!;\n    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };\n  }\n\n  const validationHeaders = cache.getValidationHeaders(url);\n\n  const headers: Record<string, string> = {\n    'User-Agent': opts.userAgent,\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n    'Accept-Language': 'en-US,en;q=0.9',\n    'Accept-Encoding': 'gzip, deflate, br, zstd',\n    // Chrome-like security headers reduce bot-detection fingerprint\n    'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n    'sec-ch-ua-mobile': '?0',\n    'sec-ch-ua-platform': '\"Windows\"',\n    'Sec-Fetch-Dest': 'document',\n    'Sec-Fetch-Mode': 'navigate',\n    'Sec-Fetch-Site': 'none',\n    'Sec-Fetch-User': '?1',\n    'Upgrade-Insecure-Requests': '1',\n    ...validationHeaders,\n    ...(opts.extraHeaders ?? {}),\n  };\n\n  let response: Response;\n  try {\n    const dispatcher = await getChromeFetchDispatcher();\n    response = await (fetch as any)(url, {\n      headers,\n      signal: AbortSignal.timeout(opts.timeoutMs),\n      redirect: 'follow',\n      dispatcher,\n    });\n  } catch (e: any) {\n    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);\n  }\n\n  const durationMs = performance.now() - t0;\n\n  // 304 Not Modified \u2014 use cached body\n  if (response.status === 304) {\n    const cached = cache.get(url);\n    if (cached) {\n      cache.markHit(url);\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };\n    }\n  }\n\n  // 429 / 503 \u2014 exponential backoff with jitter\n  if (response.status === 429 || response.status === 503) {\n    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);\n    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;\n    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));\n    const jitter = Math.random() * 1000;\n    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);\n    await new Promise(r => setTimeout(r, baseDelay + jitter));\n    return fetchWithBackoff(url, cache, opts, retryCount + 1);\n  }\n\n  // 5xx \u2014 retry up to 3 times\n  if (response.status >= 500) {\n    if (retryCount < 3) {\n      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));\n      return fetchWithBackoff(url, cache, opts, retryCount + 1);\n    }\n    if (cache.staleIfError(url)) {\n      const cached = cache.get(url)!;\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };\n    }\n  }\n\n  const contentType = response.headers.get('content-type') ?? '';\n  const body = await response.text();\n  const etag = response.headers.get('etag') ?? undefined;\n  const lastModified = response.headers.get('last-modified') ?? undefined;\n  const cacheControl = response.headers.get('cache-control') ?? undefined;\n\n  // Cache successful HTML responses\n  if (response.ok && contentType.includes('html')) {\n    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });\n  }\n\n  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };\n}\n\nexport async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {\n  const startTime = performance.now();\n  const maxScreens = opts.maxScreens ?? 100;\n  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';\n  const timeoutMs = opts.timeoutMs ?? 15_000;\n  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';\n  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;\n  const cache = new ResponseCache(cacheDir);\n  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;\n  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;\n\n  const visited = new Set<string>();\n  const queue: string[] = [opts.appUrl];\n  const queued = new Set<string>([canonicalize(opts.appUrl)]);\n  const screens: HttpScreen[] = [];\n  let cachedCount = 0;\n  let errorCount = 0;\n  let rateLimitedCount = 0;\n\n  const batchFetch = async (urls: string[]): Promise<void> => {\n    await Promise.all(urls.map(async (url) => {\n      const canonUrl = canonicalize(url);\n      if (visited.has(canonUrl)) return;\n      visited.add(canonUrl);\n\n      // Robots.txt check\n      if (robots && !(await robots.isAllowed(url).catch(() => true))) {\n        console.log(`[http-engine] robots.txt blocked: ${url}`);\n        return;\n      }\n\n      // Skip content/legal pages\n      try {\n        const { pathname } = new URL(url);\n        if (SKIP_PATH.test(pathname)) return;\n      } catch { /* skip */ }\n\n      // URL filter\n      if (urlFilterRe) {\n        try {\n          const { pathname, search } = new URL(url);\n          if (!urlFilterRe.test(pathname + search)) {\n            console.log(`[http-engine] filtered out: ${url}`);\n            return;\n          }\n        } catch { /* skip */ }\n      }\n\n      let fetchResult: FetchResult;\n      try {\n        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });\n        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;\n      } catch (err: any) {\n        console.error(`[http-engine] error fetching ${url}: ${err.message}`);\n        errorCount++;\n        return;\n      }\n\n      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML\n      if (fetchResult.statusCode >= 400) return;\n      if (fetchResult.fromCache) cachedCount++;\n\n      const { markdown, title } = extractMarkdown(fetchResult.body, url);\n      const fullMeta = extractMetadata(fetchResult.body, url);\n      const links = extractLinks(fetchResult.body, url);\n      const tables = extractTables(fetchResult.body);\n      const images = extractImages(fetchResult.body, url);\n\n      const screen: HttpScreen = {\n        url,\n        title: title ?? fullMeta.title,\n        markdown,\n        html: opts.includeRawHtml ? fetchResult.body : undefined,\n        metadata: {\n          description: fullMeta.description,\n          ogTitle: fullMeta.ogTitle,\n          ogDescription: fullMeta.ogDescription,\n          ogImage: fullMeta.ogImage,\n          canonical: fullMeta.canonical,\n          schemaTypes: fullMeta.schemaTypes,\n          lang: fullMeta.lang,\n        },\n        tables,\n        links,\n        images: images.map(i => ({ src: i.src, alt: i.alt })),\n        statusCode: fetchResult.statusCode,\n        fromCache: fetchResult.fromCache,\n        requiresBrowser: renderingPredictor.getCached(url) === 'browser',\n        durationMs: fetchResult.durationMs,\n      };\n      screens.push(screen);\n\n      // Record rendering prediction: SPA signals \u2192 browser needed; content-rich \u2192 http engine works\n      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);\n\n      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });\n\n      // Enqueue discovered links\n      for (const link of links) {\n        const c = canonicalize(link);\n        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {\n          queued.add(c);\n          if (opts.traversalStrategy === 'DFS') queue.unshift(link);\n          else queue.push(link);\n        }\n      }\n    }));\n  };\n\n  const concurrency = opts.concurrency ?? 5;\n\n  while (queue.length > 0 && screens.length < maxScreens) {\n    const batch = queue.splice(0, concurrency);\n    await batchFetch(batch);\n  }\n\n  // Output formats\n  let jsonLines: string | undefined;\n  let csv: string | undefined;\n\n  const formats = opts.outputFormats ?? [];\n  if (formats.includes('jsonlines')) {\n    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\\n');\n  }\n  if (formats.includes('csv')) {\n    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';\n    const rows = screens.map(s =>\n      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `\"${String(v).replace(/\"/g, '\"\"')}\"`).join(',')\n    );\n    csv = [header, ...rows].join('\\n');\n  }\n\n  const cacheStats = cache.stats();\n  return {\n    screens,\n    stats: {\n      total: screens.length,\n      cached: cachedCount,\n      errors: errorCount,\n      rateLimited: rateLimitedCount,\n      durationMs: performance.now() - startTime,\n      cacheHitRate: cacheStats.hitRate,\n    },\n    jsonLines,\n    csv,\n  };\n}\n"],
-  "mappings": "AAwBA,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB,eAAe,qBAAqB;AAC9D,SAAS,gBAAgB,qBAAqB;AAC9C,SAAS,0BAA0B;AAGnC,IAAI;AACJ,eAAe,2BAA2B;AACxC,MAAI,CAAC,uBAAuB;AAC1B,QAAI;AAGF,YAAM,WAAW,MAAM,OAAO,OAAO;AACrC,YAAM,aAAa,SAAS,SAAS,SAAS;AAC9C,UAAI,YAAY;AACd,gCAAwB,IAAI,WAAW,EAAE,SAAS,SAAS,CAAC;AAC5D,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAqD;AAC7D,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,4BAAwB,IAAI,MAAM;AAAA,MAChC,SAAS;AAAA,QACP,SAAS;AAAA,QACT,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,MACd;AAAA,MACA,kBAAkB;AAAA,MAClB,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AAGtF,MAAM,YAAY;AAyDlB,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AAET,eAAW,KAAK,CAAC,cAAc,cAAc,gBAAgB,eAAe,YAAY,UAAU,SAAS,OAAO,UAAU,GAAG;AAC7H,QAAE,aAAa,OAAO,CAAC;AAAA,IACzB;AACA,WAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,OAAO,EAAE;AAAA,EAC/C,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;AAEA,SAAS,aAAa,MAAc,SAA2B;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC;AAC/D,eAAW,KAAK,SAAS;AACvB,YAAM,OAAQ,EAAU,aAAa,MAAM,KAAK;AAChD,UAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,EAAG;AAC9H,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,MAAM,IAAI,EAAE;AAErC,YAAI,IAAI,IAAI,QAAQ,EAAE,WAAW,KAAK,OAAQ;AAC9C,cAAM,IAAI,aAAa,QAAQ;AAC/B,YAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAAE,eAAK,IAAI,CAAC;AAAG,gBAAM,KAAK,QAAQ;AAAA,QAAG;AAAA,MACzD,QAAQ;AAAA,MAAa;AAAA,IACvB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,SAAS,gBAAgB,MAAc,KAAyD;AAC9F,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,yBAAyB,CAAC,EAAG,CAAC,IAAY,OAAO;AACxG,UAAM,SAAS,IAAI,YAAY,UAAiB,EAAE,eAAe,GAAG,CAAC;AACrE,UAAM,UAAU,OAAO,MAAM;AAC7B,QAAI,WAAW,QAAQ,eAAe,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,IAAI;AACnF,YAAM,KAAK,SAAS,SAAS,QAAQ,WAAW,IAAI;AACpD,aAAO,EAAE,UAAU,IAAI,OAAO,QAAQ,SAAS,KAAK;AAAA,IACtD;AAAA,EACF,QAAQ;AAAA,EAAqB;AAE7B,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,sCAAsC,CAAC,EAAG,CAAC,IAAY,OAAO;AACrH,UAAM,WAAY,SAAiB,MAAM,aAAa;AACtD,UAAM,KAAK,SAAS,SAAS,QAAQ;AACrC,UAAM,UAAW,SAAiB,cAAc,OAAO;AACvD,WAAO,EAAE,UAAU,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,KAAK;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,UAAU,IAAI,OAAO,KAAK;AAAA,EACrC;AACF;AAYA,eAAe,iBACb,KACA,OACA,MACA,aAAa,GACS;AACtB,QAAM,KAAK,YAAY,IAAI;AAE3B,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,WAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,EAAE;AAAA,EAC/G;AAEA,QAAM,oBAAoB,MAAM,qBAAqB,GAAG;AAExD,QAAM,UAAkC;AAAA,IACtC,cAAc,KAAK;AAAA,IACnB,UAAU;AAAA,IACV,mBAAmB;AAAA,IACnB,mBAAmB;AAAA;AAAA,IAEnB,aAAa;AAAA,IACb,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,6BAA6B;AAAA,IAC7B,GAAG;AAAA,IACH,GAAI,KAAK,gBAAgB,CAAC;AAAA,EAC5B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,MAAM,yBAAyB;AAClD,eAAW,MAAO,MAAc,KAAK;AAAA,MACnC;AAAA,MACA,QAAQ,YAAY,QAAQ,KAAK,SAAS;AAAA,MAC1C,UAAU;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH,SAAS,GAAQ;AACf,UAAM,IAAI,MAAM,kCAAkC,GAAG,KAAK,EAAE,OAAO,EAAE;AAAA,EACvE;AAEA,QAAM,aAAa,YAAY,IAAI,IAAI;AAGvC,MAAI,SAAS,WAAW,KAAK;AAC3B,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV,YAAM,QAAQ,GAAG;AACjB,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,WAAW;AAAA,IAC5G;AAAA,EACF;AAGA,MAAI,SAAS,WAAW,OAAO,SAAS,WAAW,KAAK;AACtD,QAAI,cAAc,EAAG,OAAM,IAAI,MAAM,+CAA+C,GAAG,EAAE;AACzF,UAAM,aAAa,SAAS,SAAS,QAAQ,IAAI,aAAa,KAAK,GAAG,KAAK;AAC3E,UAAM,YAAY,aAAa,IAAI,aAAa,MAAO,KAAK,IAAI,KAAQ,MAAQ,KAAK,IAAI,GAAG,UAAU,CAAC;AACvG,UAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAQ,IAAI,iBAAiB,SAAS,MAAM,OAAO,GAAG,uBAAkB,KAAK,MAAM,YAAY,MAAM,CAAC,aAAa,aAAa,CAAC,KAAK;AACtI,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,YAAY,MAAM,CAAC;AACxD,WAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,EAC1D;AAGA,MAAI,SAAS,UAAU,KAAK;AAC1B,QAAI,aAAa,GAAG;AAClB,YAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,OAAS,aAAa,EAAE,CAAC;AAC9D,aAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,CAAC;AAAA,IAC1D;AACA,QAAI,MAAM,aAAa,GAAG,GAAG;AAC3B,YAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,YAAY,IAAI,IAAI,GAAG;AAAA,IACpI;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAM,OAAO,MAAM,SAAS,KAAK;AACjC,QAAM,OAAO,SAAS,QAAQ,IAAI,MAAM,KAAK;AAC7C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAC9D,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAG9D,MAAI,SAAS,MAAM,YAAY,SAAS,MAAM,GAAG;AAC/C,UAAM,IAAI,KAAK,EAAE,KAAK,MAAM,cAAc,cAAc,MAAM,aAAa,YAAY,SAAS,OAAO,CAAC;AAAA,EAC1G;AAEA,SAAO,EAAE,MAAM,YAAY,SAAS,QAAQ,aAAa,WAAW,OAAO,YAAY,MAAM,aAAa;AAC5G;AAEA,eAAsB,UAAU,MAA+C;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,WAAW,KAAK,YAAY,QAAQ,IAAI,qBAAqB;AACnE,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,qBAAqB,QAAQ,IAAI,sBAAsB;AAC7D,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,QAAQ,IAAI,cAAc,QAAQ;AACxC,QAAM,SAAS,KAAK,qBAAqB,QAAQ,IAAI,YAAY,IAAI;AACrE,QAAM,cAAc,KAAK,aAAa,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,OAAO,KAAK,WAAY,GAAG;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAM;AAAA,EAAE,GAAG,IAAI;AAE7H,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,QAAkB,CAAC,KAAK,MAAM;AACpC,QAAM,SAAS,oBAAI,IAAY,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;AAC1D,QAAM,UAAwB,CAAC;AAC/B,MAAI,cAAc;AAClB,MAAI,aAAa;AACjB,MAAI,mBAAmB;AAEvB,QAAM,aAAa,OAAO,SAAkC;AAC1D,UAAM,QAAQ,IAAI,KAAK,IAAI,OAAO,QAAQ;AACxC,YAAM,WAAW,aAAa,GAAG;AACjC,UAAI,QAAQ,IAAI,QAAQ,EAAG;AAC3B,cAAQ,IAAI,QAAQ;AAGpB,UAAI,UAAU,CAAE,MAAM,OAAO,UAAU,GAAG,EAAE,MAAM,MAAM,IAAI,GAAI;AAC9D,gBAAQ,IAAI,qCAAqC,GAAG,EAAE;AACtD;AAAA,MACF;AAGA,UAAI;AACF,cAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,YAAI,UAAU,KAAK,QAAQ,EAAG;AAAA,MAChC,QAAQ;AAAA,MAAa;AAGrB,UAAI,aAAa;AACf,YAAI;AACF,gBAAM,EAAE,UAAU,OAAO,IAAI,IAAI,IAAI,GAAG;AACxC,cAAI,CAAC,YAAY,KAAK,WAAW,MAAM,GAAG;AACxC,oBAAQ,IAAI,+BAA+B,GAAG,EAAE;AAChD;AAAA,UACF;AAAA,QACF,QAAQ;AAAA,QAAa;AAAA,MACvB;AAEA,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,iBAAiB,KAAK,OAAO,EAAE,WAAW,WAAW,cAAc,KAAK,QAAQ,CAAC;AACrG,YAAI,YAAY,eAAe,OAAO,YAAY,eAAe,IAAK;AAAA,MACxE,SAAS,KAAU;AACjB,gBAAQ,MAAM,gCAAgC,GAAG,KAAK,IAAI,OAAO,EAAE;AACnE;AACA;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,YAAY,SAAS,MAAM,EAAG;AAC/C,UAAI,YAAY,cAAc,IAAK;AACnC,UAAI,YAAY,UAAW;AAE3B,YAAM,EAAE,UAAU,MAAM,IAAI,gBAAgB,YAAY,MAAM,GAAG;AACjE,YAAM,WAAW,gBAAgB,YAAY,MAAM,GAAG;AACtD,YAAM,QAAQ,aAAa,YAAY,MAAM,GAAG;AAChD,YAAM,SAAS,cAAc,YAAY,IAAI;AAC7C,YAAM,SAAS,cAAc,YAAY,MAAM,GAAG;AAElD,YAAM,SAAqB;AAAA,QACzB;AAAA,QACA,OAAO,SAAS,SAAS;AAAA,QACzB;AAAA,QACA,MAAM,KAAK,iBAAiB,YAAY,OAAO;AAAA,QAC/C,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,SAAS,SAAS;AAAA,UAClB,eAAe,SAAS;AAAA,UACxB,SAAS,SAAS;AAAA,UAClB,WAAW,SAAS;AAAA,UACpB,aAAa,SAAS;AAAA,UACtB,MAAM,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE;AAAA,QACpD,YAAY,YAAY;AAAA,QACxB,WAAW,YAAY;AAAA,QACvB,iBAAiB,mBAAmB,UAAU,GAAG,MAAM;AAAA,QACvD,YAAY,YAAY;AAAA,MAC1B;AACA,cAAQ,KAAK,MAAM;AAGnB,yBAAmB,QAAQ,KAAK,YAAY,MAAM,YAAY,UAAU;AAExE,WAAK,aAAa,EAAE,KAAK,cAAc,QAAQ,QAAQ,OAAO,YAAY,QAAQ,YAAY,UAAU,CAAC;AAGzG,iBAAW,QAAQ,OAAO;AACxB,cAAM,IAAI,aAAa,IAAI;AAC3B,YAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,QAAQ,SAAS,MAAM,SAAS,aAAa,GAAG;AACvF,iBAAO,IAAI,CAAC;AACZ,cAAI,KAAK,sBAAsB,MAAO,OAAM,QAAQ,IAAI;AAAA,cACnD,OAAM,KAAK,IAAI;AAAA,QACtB;AAAA,MACF;AAAA,IACF,CAAC,CAAC;AAAA,EACJ;AAEA,QAAM,cAAc,KAAK,eAAe;AAExC,SAAO,MAAM,SAAS,KAAK,QAAQ,SAAS,YAAY;AACtD,UAAM,QAAQ,MAAM,OAAO,GAAG,WAAW;AACzC,UAAM,WAAW,KAAK;AAAA,EACxB;AAGA,MAAI;AACJ,MAAI;AAEJ,QAAM,UAAU,KAAK,iBAAiB,CAAC;AACvC,MAAI,QAAQ,SAAS,WAAW,GAAG;AACjC,gBAAY,QAAQ,IAAI,OAAK,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,UAAU,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACpI;AACA,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS;AACf,UAAM,OAAO,QAAQ;AAAA,MAAI,OACvB,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,SAAS,eAAe,IAAI,EAAE,SAAS,WAAW,IAAI,EAAE,SAAS,YAAY,KAAK,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,OAAK,IAAI,OAAO,CAAC,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG;AAAA,IAC/N;AACA,UAAM,CAAC,QAAQ,GAAG,IAAI,EAAE,KAAK,IAAI;AAAA,EACnC;AAEA,QAAM,aAAa,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,OAAO,QAAQ;AAAA,MACf,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC,cAAc,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
+  "sourcesContent": ["/**\n * HTTP-only crawl engine \u2014 10-50x faster than browser for SSR/static sites.\n *\n * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.\n * No browser overhead. Falls back to browser engine on JS-heavy sites.\n *\n * Features:\n *   - HTTP/2 native (Node 20 built-in fetch)\n *   - ETag / Last-Modified response caching\n *   - 429 / 503 exponential backoff with jitter\n *   - BFS / DFS traversal\n *   - URL regex filtering\n *   - Robots.txt compliance\n *   - Link discovery (href, action, src)\n *   - Markdown extraction (Readability + Turndown)\n *   - OG / schema.org / meta extraction\n *   - Table \u2192 JSON extraction\n *   - JSON Lines, CSV, raw HTML output formats\n *   - Streaming progress via onProgress callback\n */\n\nimport { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { parseHTML } from 'linkedom';\nimport { RobotsCache } from './robots-txt.js';\nimport { ResponseCache } from './response-cache.js';\nimport { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';\nimport { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';\nimport { renderingPredictor } from './engine-router.js';\n\n// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback\nlet chromeFetchDispatcher: any;\nasync function getChromeFetchDispatcher() {\n  if (!chromeFetchDispatcher) {\n    try {\n      // impit provides native Chrome BoringSSL TLS fingerprint \u2014 true JA3/JA4 match\n      // @ts-ignore \u2014 impit is an optional dependency\n      const impitMod = await import('impit');\n      const ImpitClass = impitMod.Impit ?? impitMod.default;\n      if (ImpitClass) {\n        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });\n        return chromeFetchDispatcher;\n      }\n    } catch { /* impit not installed \u2014 fall through to undici */ }\n    const { Agent } = await import('undici');\n    chromeFetchDispatcher = new Agent({\n      connect: {\n        ciphers: CHROME_CIPHERS,\n        ecdhCurve: CHROME_CURVES,\n        minVersion: 'TLSv1.2' as any,\n        maxVersion: 'TLSv1.3' as any,\n      },\n      keepAliveTimeout: 10_000,\n      keepAliveMaxTimeout: 30_000,\n    });\n  }\n  return chromeFetchDispatcher;\n}\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\n\n// Pages unlikely to have testable UI \u2014 skip from HTTP crawl queue\nconst _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA\n  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)\n  : [];\nconst SKIP_PATH = _skipPathExtra.length > 0\n  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')\n  : /\\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;\n\nasync function getHttpConfig() {\n  const { getSystemConfig } = await import('@detiq/core');\n  const cfg = await getSystemConfig('crawl_tuning');\n  return {\n    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),\n    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),\n    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),\n    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),\n    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),\n  };\n}\n\nexport type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';\n\nexport interface HttpScreen {\n  url: string;\n  title: string | null;\n  markdown: string;\n  html?: string;\n  metadata: {\n    description: string | null;\n    ogTitle: string | null;\n    ogDescription: string | null;\n    ogImage: string | null;\n    canonical: string | null;\n    schemaTypes: string[];\n    lang: string | null;\n  };\n  tables: Array<Record<string, string>[]>;\n  links: string[];\n  images: Array<{ src: string; alt: string }>;\n  statusCode: number;\n  fromCache: boolean;\n  requiresBrowser?: boolean;\n  durationMs: number;\n}\n\nexport interface HttpCrawlOpts {\n  appUrl: string;\n  maxScreens?: number;            // default 100\n  urlFilter?: string;             // regex pattern \u2014 only crawl matching URLs\n  traversalStrategy?: 'BFS' | 'DFS'; // default BFS\n  outputFormats?: HttpOutputFormat[];\n  cacheDir?: string;              // default '/tmp/zeta-cache'\n  userAgent?: string;\n  headers?: Record<string, string>;\n  respectRobotsTxt?: boolean;     // default true\n  includeRawHtml?: boolean;\n  timeoutMs?: number;             // per-request timeout, default 15000\n  concurrency?: number;           // default 5 parallel fetches\n  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;\n}\n\nexport interface HttpCrawlResult {\n  screens: HttpScreen[];\n  stats: {\n    total: number;\n    cached: number;\n    errors: number;\n    rateLimited: number;\n    durationMs: number;\n    cacheHitRate: number;\n  };\n  jsonLines?: string;\n  csv?: string;\n}\n\nfunction canonicalize(url: string): string {\n  try {\n    const u = new URL(url);\n    u.hash = '';\n    // Strip tracking params\n    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {\n      u.searchParams.delete(p);\n    }\n    return u.href.toLowerCase().replace(/\\/$/, '');\n  } catch {\n    return url.toLowerCase();\n  }\n}\n\nfunction extractLinks(html: string, baseUrl: string): string[] {\n  const links: string[] = [];\n  const seen = new Set<string>();\n  try {\n    const base = new URL(baseUrl);\n    const { document } = parseHTML(html);\n    const anchors = Array.from(document.querySelectorAll('a[href]'));\n    for (const a of anchors) {\n      const href = (a as any).getAttribute('href') ?? '';\n      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;\n      try {\n        const resolved = new URL(href, base).href;\n        // Same origin only\n        if (new URL(resolved).origin !== base.origin) continue;\n        const c = canonicalize(resolved);\n        if (!seen.has(c)) { seen.add(c); links.push(resolved); }\n      } catch { /* skip */ }\n    }\n  } catch { /* non-fatal */ }\n  return links;\n}\n\nfunction extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {\n  try {\n    const { document } = parseHTML(html);\n    // Remove script/style tags before Readability\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();\n    const reader = new Readability(document as any, { charThreshold: 50 });\n    const article = reader.parse();\n    if (article && article.textContent && article.textContent.split(/\\s+/).length >= 30) {\n      const md = turndown.turndown(article.content ?? html);\n      return { markdown: md, title: article.title ?? null };\n    }\n  } catch { /* fall through */ }\n  // Full-page fallback\n  try {\n    const { document } = parseHTML(html);\n    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();\n    const bodyHtml = (document as any).body?.innerHTML ?? html;\n    const md = turndown.turndown(bodyHtml);\n    const titleEl = (document as any).querySelector('title');\n    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };\n  } catch {\n    return { markdown: '', title: null };\n  }\n}\n\ninterface FetchResult {\n  body: string;\n  statusCode: number;\n  contentType: string;\n  fromCache: boolean;\n  durationMs: number;\n  etag?: string;\n  lastModified?: string;\n}\n\nasync function fetchWithBackoff(\n  url: string,\n  cache: ResponseCache,\n  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },\n  retryCount = 0,\n  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>\n): Promise<FetchResult> {\n  if (!httpCfg) httpCfg = await getHttpConfig();\n  const t0 = performance.now();\n\n  if (cache.isFresh(url)) {\n    const cached = cache.get(url)!;\n    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };\n  }\n\n  const validationHeaders = cache.getValidationHeaders(url);\n\n  const headers: Record<string, string> = {\n    'User-Agent': opts.userAgent,\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n    'Accept-Language': 'en-US,en;q=0.9',\n    'Accept-Encoding': 'gzip, deflate, br, zstd',\n    // Chrome-like security headers reduce bot-detection fingerprint\n    'sec-ch-ua': `\"Google Chrome\";v=\"${process.env.CRAWLER_CHROME_VERSION ?? '120'}\", \"Chromium\";v=\"${process.env.CRAWLER_CHROME_VERSION ?? '120'}\", \"Not-A.Brand\";v=\"99\"`,\n    'sec-ch-ua-mobile': '?0',\n    'sec-ch-ua-platform': '\"Windows\"',\n    'Sec-Fetch-Dest': 'document',\n    'Sec-Fetch-Mode': 'navigate',\n    'Sec-Fetch-Site': 'none',\n    'Sec-Fetch-User': '?1',\n    'Upgrade-Insecure-Requests': '1',\n    ...validationHeaders,\n    ...(opts.extraHeaders ?? {}),\n  };\n\n  let response: Response;\n  try {\n    const dispatcher = await getChromeFetchDispatcher();\n    response = await (fetch as any)(url, {\n      headers,\n      signal: AbortSignal.timeout(opts.timeoutMs),\n      redirect: 'follow',\n      dispatcher,\n    });\n  } catch (e: any) {\n    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);\n  }\n\n  const durationMs = performance.now() - t0;\n\n  // 304 Not Modified \u2014 use cached body\n  if (response.status === 304) {\n    const cached = cache.get(url);\n    if (cached) {\n      cache.markHit(url);\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };\n    }\n  }\n\n  // 429 / 503 \u2014 exponential backoff with jitter\n  if (response.status === 429 || response.status === 503) {\n    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);\n    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;\n    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));\n    const jitter = Math.random() * 1000;\n    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);\n    await new Promise(r => setTimeout(r, baseDelay + jitter));\n    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);\n  }\n\n  // 5xx \u2014 retry up to 3 times\n  if (response.status >= 500) {\n    if (retryCount < httpCfg.retry5xx) {\n      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));\n      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);\n    }\n    if (cache.staleIfError(url)) {\n      const cached = cache.get(url)!;\n      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };\n    }\n  }\n\n  const contentType = response.headers.get('content-type') ?? '';\n  const body = await response.text();\n  const etag = response.headers.get('etag') ?? undefined;\n  const lastModified = response.headers.get('last-modified') ?? undefined;\n  const cacheControl = response.headers.get('cache-control') ?? undefined;\n\n  // Cache successful HTML responses\n  if (response.ok && contentType.includes('html')) {\n    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });\n  }\n\n  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };\n}\n\nexport async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {\n  const startTime = performance.now();\n  const httpCfg = await getHttpConfig();\n  const maxScreens = opts.maxScreens ?? 100;\n  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';\n  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;\n  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';\n  const cache = new ResponseCache(cacheDir);\n  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;\n  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;\n\n  const visited = new Set<string>();\n  const queue: string[] = [opts.appUrl];\n  const queued = new Set<string>([canonicalize(opts.appUrl)]);\n  const screens: HttpScreen[] = [];\n  let cachedCount = 0;\n  let errorCount = 0;\n  let rateLimitedCount = 0;\n\n  const batchFetch = async (urls: string[]): Promise<void> => {\n    await Promise.all(urls.map(async (url) => {\n      const canonUrl = canonicalize(url);\n      if (visited.has(canonUrl)) return;\n      visited.add(canonUrl);\n\n      // Robots.txt check\n      if (robots && !(await robots.isAllowed(url).catch(() => true))) {\n        console.log(`[http-engine] robots.txt blocked: ${url}`);\n        return;\n      }\n\n      // Skip content/legal pages\n      try {\n        const { pathname } = new URL(url);\n        if (SKIP_PATH.test(pathname)) return;\n      } catch { /* skip */ }\n\n      // URL filter\n      if (urlFilterRe) {\n        try {\n          const { pathname, search } = new URL(url);\n          if (!urlFilterRe.test(pathname + search)) {\n            console.log(`[http-engine] filtered out: ${url}`);\n            return;\n          }\n        } catch { /* skip */ }\n      }\n\n      let fetchResult: FetchResult;\n      try {\n        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);\n        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;\n      } catch (err: any) {\n        console.error(`[http-engine] error fetching ${url}: ${err.message}`);\n        errorCount++;\n        return;\n      }\n\n      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML\n      if (fetchResult.statusCode >= 400) return;\n      if (fetchResult.fromCache) cachedCount++;\n\n      const { markdown, title } = extractMarkdown(fetchResult.body, url);\n      const fullMeta = extractMetadata(fetchResult.body, url);\n      const links = extractLinks(fetchResult.body, url);\n      const tables = extractTables(fetchResult.body);\n      const images = extractImages(fetchResult.body, url);\n\n      const screen: HttpScreen = {\n        url,\n        title: title ?? fullMeta.title,\n        markdown,\n        html: opts.includeRawHtml ? fetchResult.body : undefined,\n        metadata: {\n          description: fullMeta.description,\n          ogTitle: fullMeta.ogTitle,\n          ogDescription: fullMeta.ogDescription,\n          ogImage: fullMeta.ogImage,\n          canonical: fullMeta.canonical,\n          schemaTypes: fullMeta.schemaTypes,\n          lang: fullMeta.lang,\n        },\n        tables,\n        links,\n        images: images.map(i => ({ src: i.src, alt: i.alt })),\n        statusCode: fetchResult.statusCode,\n        fromCache: fetchResult.fromCache,\n        requiresBrowser: renderingPredictor.getCached(url) === 'browser',\n        durationMs: fetchResult.durationMs,\n      };\n      screens.push(screen);\n\n      // Record rendering prediction: SPA signals \u2192 browser needed; content-rich \u2192 http engine works\n      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);\n\n      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });\n\n      // Enqueue discovered links\n      for (const link of links) {\n        const c = canonicalize(link);\n        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {\n          queued.add(c);\n          if (opts.traversalStrategy === 'DFS') queue.unshift(link);\n          else queue.push(link);\n        }\n      }\n    }));\n  };\n\n  const concurrency = opts.concurrency ?? httpCfg.concurrency;\n\n  while (queue.length > 0 && screens.length < maxScreens) {\n    const batch = queue.splice(0, concurrency);\n    await batchFetch(batch);\n  }\n\n  // Output formats\n  let jsonLines: string | undefined;\n  let csv: string | undefined;\n\n  const formats = opts.outputFormats ?? [];\n  if (formats.includes('jsonlines')) {\n    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\\n');\n  }\n  if (formats.includes('csv')) {\n    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';\n    const rows = screens.map(s =>\n      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `\"${String(v).replace(/\"/g, '\"\"')}\"`).join(',')\n    );\n    csv = [header, ...rows].join('\\n');\n  }\n\n  const cacheStats = cache.stats();\n  return {\n    screens,\n    stats: {\n      total: screens.length,\n      cached: cachedCount,\n      errors: errorCount,\n      rateLimited: rateLimitedCount,\n      durationMs: performance.now() - startTime,\n      cacheHitRate: cacheStats.hitRate,\n    },\n    jsonLines,\n    csv,\n  };\n}\n"],
+  "mappings": "AAwBA,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAC5B,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB,eAAe,qBAAqB;AAC9D,SAAS,gBAAgB,qBAAqB;AAC9C,SAAS,0BAA0B;AAGnC,IAAI;AACJ,eAAe,2BAA2B;AACxC,MAAI,CAAC,uBAAuB;AAC1B,QAAI;AAGF,YAAM,WAAW,MAAM,OAAO,OAAO;AACrC,YAAM,aAAa,SAAS,SAAS,SAAS;AAC9C,UAAI,YAAY;AACd,gCAAwB,IAAI,WAAW,EAAE,SAAS,SAAS,CAAC;AAC5D,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAqD;AAC7D,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,4BAAwB,IAAI,MAAM;AAAA,MAChC,SAAS;AAAA,QACP,SAAS;AAAA,QACT,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,MACd;AAAA,MACA,kBAAkB;AAAA,MAClB,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AAGtF,MAAM,iBAAiB,QAAQ,IAAI,wBAC/B,QAAQ,IAAI,sBAAsB,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO,IAChF,CAAC;AACL,MAAM,YAAY,eAAe,SAAS,IACtC,IAAI,OAAO,8JAA8J,eAAe,KAAK,GAAG,CAAC,aAAa,GAAG,IACjN;AAEJ,eAAe,gBAAgB;AAC7B,QAAM,EAAE,gBAAgB,IAAI,MAAM,OAAO,aAAa;AACtD,QAAM,MAAM,MAAM,gBAAgB,cAAc;AAChD,SAAO;AAAA,IACL,YAAc,SAAS,OAAO,KAAK,kBAAqB,GAAG,GAAO,EAAE;AAAA,IACpE,cAAc,SAAS,OAAO,KAAK,oBAAqB,OAAO,GAAG,EAAE;AAAA,IACpE,UAAc,SAAS,OAAO,KAAK,qBAAqB,GAAG,GAAO,EAAE;AAAA,IACpE,WAAc,SAAS,OAAO,KAAK,iBAAqB,OAAO,GAAG,EAAE;AAAA,IACpE,aAAc,SAAS,OAAO,KAAK,mBAAqB,GAAG,GAAO,EAAE;AAAA,EACtE;AACF;AAyDA,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AAET,eAAW,KAAK,CAAC,cAAc,cAAc,gBAAgB,eAAe,YAAY,UAAU,SAAS,OAAO,UAAU,GAAG;AAC7H,QAAE,aAAa,OAAO,CAAC;AAAA,IACzB;AACA,WAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,OAAO,EAAE;AAAA,EAC/C,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;AAEA,SAAS,aAAa,MAAc,SAA2B;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC;AAC/D,eAAW,KAAK,SAAS;AACvB,YAAM,OAAQ,EAAU,aAAa,MAAM,KAAK;AAChD,UAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,EAAG;AAC9H,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,MAAM,IAAI,EAAE;AAErC,YAAI,IAAI,IAAI,QAAQ,EAAE,WAAW,KAAK,OAAQ;AAC9C,cAAM,IAAI,aAAa,QAAQ;AAC/B,YAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAAE,eAAK,IAAI,CAAC;AAAG,gBAAM,KAAK,QAAQ;AAAA,QAAG;AAAA,MACzD,QAAQ;AAAA,MAAa;AAAA,IACvB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,SAAS,gBAAgB,MAAc,KAAyD;AAC9F,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,yBAAyB,CAAC,EAAG,CAAC,IAAY,OAAO;AACxG,UAAM,SAAS,IAAI,YAAY,UAAiB,EAAE,eAAe,GAAG,CAAC;AACrE,UAAM,UAAU,OAAO,MAAM;AAC7B,QAAI,WAAW,QAAQ,eAAe,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,IAAI;AACnF,YAAM,KAAK,SAAS,SAAS,QAAQ,WAAW,IAAI;AACpD,aAAO,EAAE,UAAU,IAAI,OAAO,QAAQ,SAAS,KAAK;AAAA,IACtD;AAAA,EACF,QAAQ;AAAA,EAAqB;AAE7B,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,eAAW,OAAO,MAAM,KAAK,SAAS,iBAAiB,sCAAsC,CAAC,EAAG,CAAC,IAAY,OAAO;AACrH,UAAM,WAAY,SAAiB,MAAM,aAAa;AACtD,UAAM,KAAK,SAAS,SAAS,QAAQ;AACrC,UAAM,UAAW,SAAiB,cAAc,OAAO;AACvD,WAAO,EAAE,UAAU,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,KAAK;AAAA,EACrE,QAAQ;AACN,WAAO,EAAE,UAAU,IAAI,OAAO,KAAK;AAAA,EACrC;AACF;AAYA,eAAe,iBACb,KACA,OACA,MACA,aAAa,GACb,SACsB;AACtB,MAAI,CAAC,QAAS,WAAU,MAAM,cAAc;AAC5C,QAAM,KAAK,YAAY,IAAI;AAE3B,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,WAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,EAAE;AAAA,EAC/G;AAEA,QAAM,oBAAoB,MAAM,qBAAqB,GAAG;AAExD,QAAM,UAAkC;AAAA,IACtC,cAAc,KAAK;AAAA,IACnB,UAAU;AAAA,IACV,mBAAmB;AAAA,IACnB,mBAAmB;AAAA;AAAA,IAEnB,aAAa,sBAAsB,QAAQ,IAAI,0BAA0B,KAAK,oBAAoB,QAAQ,IAAI,0BAA0B,KAAK;AAAA,IAC7I,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,6BAA6B;AAAA,IAC7B,GAAG;AAAA,IACH,GAAI,KAAK,gBAAgB,CAAC;AAAA,EAC5B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,MAAM,yBAAyB;AAClD,eAAW,MAAO,MAAc,KAAK;AAAA,MACnC;AAAA,MACA,QAAQ,YAAY,QAAQ,KAAK,SAAS;AAAA,MAC1C,UAAU;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH,SAAS,GAAQ;AACf,UAAM,IAAI,MAAM,kCAAkC,GAAG,KAAK,EAAE,OAAO,EAAE;AAAA,EACvE;AAEA,QAAM,aAAa,YAAY,IAAI,IAAI;AAGvC,MAAI,SAAS,WAAW,KAAK;AAC3B,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV,YAAM,QAAQ,GAAG;AACjB,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,WAAW;AAAA,IAC5G;AAAA,EACF;AAGA,MAAI,SAAS,WAAW,OAAO,SAAS,WAAW,KAAK;AACtD,QAAI,cAAc,QAAQ,WAAY,OAAM,IAAI,MAAM,oCAAoC,QAAQ,UAAU,aAAa,GAAG,EAAE;AAC9H,UAAM,aAAa,SAAS,SAAS,QAAQ,IAAI,aAAa,KAAK,GAAG,KAAK;AAC3E,UAAM,YAAY,aAAa,IAAI,aAAa,MAAO,KAAK,IAAI,QAAQ,cAAc,MAAQ,KAAK,IAAI,GAAG,UAAU,CAAC;AACrH,UAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAQ,IAAI,iBAAiB,SAAS,MAAM,OAAO,GAAG,uBAAkB,KAAK,MAAM,YAAY,MAAM,CAAC,aAAa,aAAa,CAAC,IAAI,QAAQ,UAAU,GAAG;AAC1J,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,YAAY,MAAM,CAAC;AACxD,WAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,GAAG,OAAO;AAAA,EACnE;AAGA,MAAI,SAAS,UAAU,KAAK;AAC1B,QAAI,aAAa,QAAQ,UAAU;AACjC,YAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,OAAS,aAAa,EAAE,CAAC;AAC9D,aAAO,iBAAiB,KAAK,OAAO,MAAM,aAAa,GAAG,OAAO;AAAA,IACnE;AACA,QAAI,MAAM,aAAa,GAAG,GAAG;AAC3B,YAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,aAAO,EAAE,MAAM,OAAO,MAAM,YAAY,KAAK,aAAa,OAAO,aAAa,WAAW,MAAM,YAAY,YAAY,IAAI,IAAI,GAAG;AAAA,IACpI;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAM,OAAO,MAAM,SAAS,KAAK;AACjC,QAAM,OAAO,SAAS,QAAQ,IAAI,MAAM,KAAK;AAC7C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAC9D,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe,KAAK;AAG9D,MAAI,SAAS,MAAM,YAAY,SAAS,MAAM,GAAG;AAC/C,UAAM,IAAI,KAAK,EAAE,KAAK,MAAM,cAAc,cAAc,MAAM,aAAa,YAAY,SAAS,OAAO,CAAC;AAAA,EAC1G;AAEA,SAAO,EAAE,MAAM,YAAY,SAAS,QAAQ,aAAa,WAAW,OAAO,YAAY,MAAM,aAAa;AAC5G;AAEA,eAAsB,UAAU,MAA+C;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,UAAU,MAAM,cAAc;AACpC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,YAAY,KAAK,aAAa,QAAQ;AAC5C,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,QAAQ,IAAI,cAAc,QAAQ;AACxC,QAAM,SAAS,KAAK,qBAAqB,QAAQ,IAAI,YAAY,IAAI;AACrE,QAAM,cAAc,KAAK,aAAa,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,OAAO,KAAK,WAAY,GAAG;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAM;AAAA,EAAE,GAAG,IAAI;AAE7H,QAAM,UAAU,oBAAI,IAAY;AAChC,QAAM,QAAkB,CAAC,KAAK,MAAM;AACpC,QAAM,SAAS,oBAAI,IAAY,CAAC,aAAa,KAAK,MAAM,CAAC,CAAC;AAC1D,QAAM,UAAwB,CAAC;AAC/B,MAAI,cAAc;AAClB,MAAI,aAAa;AACjB,MAAI,mBAAmB;AAEvB,QAAM,aAAa,OAAO,SAAkC;AAC1D,UAAM,QAAQ,IAAI,KAAK,IAAI,OAAO,QAAQ;AACxC,YAAM,WAAW,aAAa,GAAG;AACjC,UAAI,QAAQ,IAAI,QAAQ,EAAG;AAC3B,cAAQ,IAAI,QAAQ;AAGpB,UAAI,UAAU,CAAE,MAAM,OAAO,UAAU,GAAG,EAAE,MAAM,MAAM,IAAI,GAAI;AAC9D,gBAAQ,IAAI,qCAAqC,GAAG,EAAE;AACtD;AAAA,MACF;AAGA,UAAI;AACF,cAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,YAAI,UAAU,KAAK,QAAQ,EAAG;AAAA,MAChC,QAAQ;AAAA,MAAa;AAGrB,UAAI,aAAa;AACf,YAAI;AACF,gBAAM,EAAE,UAAU,OAAO,IAAI,IAAI,IAAI,GAAG;AACxC,cAAI,CAAC,YAAY,KAAK,WAAW,MAAM,GAAG;AACxC,oBAAQ,IAAI,+BAA+B,GAAG,EAAE;AAChD;AAAA,UACF;AAAA,QACF,QAAQ;AAAA,QAAa;AAAA,MACvB;AAEA,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,iBAAiB,KAAK,OAAO,EAAE,WAAW,WAAW,cAAc,KAAK,QAAQ,GAAG,GAAG,OAAO;AACjH,YAAI,YAAY,eAAe,OAAO,YAAY,eAAe,IAAK;AAAA,MACxE,SAAS,KAAU;AACjB,gBAAQ,MAAM,gCAAgC,GAAG,KAAK,IAAI,OAAO,EAAE;AACnE;AACA;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,YAAY,SAAS,MAAM,EAAG;AAC/C,UAAI,YAAY,cAAc,IAAK;AACnC,UAAI,YAAY,UAAW;AAE3B,YAAM,EAAE,UAAU,MAAM,IAAI,gBAAgB,YAAY,MAAM,GAAG;AACjE,YAAM,WAAW,gBAAgB,YAAY,MAAM,GAAG;AACtD,YAAM,QAAQ,aAAa,YAAY,MAAM,GAAG;AAChD,YAAM,SAAS,cAAc,YAAY,IAAI;AAC7C,YAAM,SAAS,cAAc,YAAY,MAAM,GAAG;AAElD,YAAM,SAAqB;AAAA,QACzB;AAAA,QACA,OAAO,SAAS,SAAS;AAAA,QACzB;AAAA,QACA,MAAM,KAAK,iBAAiB,YAAY,OAAO;AAAA,QAC/C,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,SAAS,SAAS;AAAA,UAClB,eAAe,SAAS;AAAA,UACxB,SAAS,SAAS;AAAA,UAClB,WAAW,SAAS;AAAA,UACpB,aAAa,SAAS;AAAA,UACtB,MAAM,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE;AAAA,QACpD,YAAY,YAAY;AAAA,QACxB,WAAW,YAAY;AAAA,QACvB,iBAAiB,mBAAmB,UAAU,GAAG,MAAM;AAAA,QACvD,YAAY,YAAY;AAAA,MAC1B;AACA,cAAQ,KAAK,MAAM;AAGnB,yBAAmB,QAAQ,KAAK,YAAY,MAAM,YAAY,UAAU;AAExE,WAAK,aAAa,EAAE,KAAK,cAAc,QAAQ,QAAQ,OAAO,YAAY,QAAQ,YAAY,UAAU,CAAC;AAGzG,iBAAW,QAAQ,OAAO;AACxB,cAAM,IAAI,aAAa,IAAI;AAC3B,YAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,QAAQ,SAAS,MAAM,SAAS,aAAa,GAAG;AACvF,iBAAO,IAAI,CAAC;AACZ,cAAI,KAAK,sBAAsB,MAAO,OAAM,QAAQ,IAAI;AAAA,cACnD,OAAM,KAAK,IAAI;AAAA,QACtB;AAAA,MACF;AAAA,IACF,CAAC,CAAC;AAAA,EACJ;AAEA,QAAM,cAAc,KAAK,eAAe,QAAQ;AAEhD,SAAO,MAAM,SAAS,KAAK,QAAQ,SAAS,YAAY;AACtD,UAAM,QAAQ,MAAM,OAAO,GAAG,WAAW;AACzC,UAAM,WAAW,KAAK;AAAA,EACxB;AAGA,MAAI;AACJ,MAAI;AAEJ,QAAM,UAAU,KAAK,iBAAiB,CAAC;AACvC,MAAI,QAAQ,SAAS,WAAW,GAAG;AACjC,gBAAY,QAAQ,IAAI,OAAK,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,UAAU,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACpI;AACA,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,UAAM,SAAS;AACf,UAAM,OAAO,QAAQ;AAAA,MAAI,OACvB,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,SAAS,eAAe,IAAI,EAAE,SAAS,WAAW,IAAI,EAAE,SAAS,YAAY,KAAK,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,KAAK,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,OAAK,IAAI,OAAO,CAAC,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG;AAAA,IAC/N;AACA,UAAM,CAAC,QAAQ,GAAG,IAAI,EAAE,KAAK,IAAI;AAAA,EACnC;AAEA,QAAM,aAAa,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,OAAO,QAAQ;AAAA,MACf,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC,cAAc,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/index.js

@@ -25,7 +25,20 @@ import { TieredProxyManager } from "./proxy.js";
 import { RedisCrawlFrontier, isRedisFrontierEnabled } from "./redis-frontier.js";
 import { startSpan, endSpan, withSpan, increment, flushMetrics } from "./telemetry.js";
 import { analyzeFailure } from "./failure-analysis-agent.js";
-import { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from "./locator-extractor.js";
+import { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from "./locator-extractor.js";
+export * from "./dropdown-enumerator.js";
+export * from "./crud-interaction.js";
+export * from "./filter-pagination-crawler.js";
+export * from "./multi-context-handler.js";
+export * from "./advanced-ui-probes.js";
+export * from "./state-graph.js";
+export * from "./nested-window-crawler.js";
+export * from "./shadow-dom-crawler.js";
+export * from "./network-action-correlator.js";
+export * from "./crawl-session-manager.js";
+export * from "./accessibility-tree-crawler.js";
+export * from "./performance-metrics.js";
+export * from "./service-worker-interceptor.js";
 export {
   CHROME_TLS_HEADERS,
   RedisCrawlFrontier,
@@ -37,6 +50,7 @@ export {
   completeLiveView,
   createChromeTlsAgent,
   createRedisQueue,
+  detectAuthWallLocators,
   discoverProjectUrls,
   discoverSitemapUrls,
   elementToPlaywrightLocator,

package/dist/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/index.ts"],
-  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\nexport { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';\nexport type { PlaywrightLocator } from './locator-extractor.js';\n"],
-  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;AAE/B,SAAS,4BAA4B,8BAA8B,+BAA+B;",
+  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\nexport { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';\nexport type { PlaywrightLocator } from './locator-extractor.js';\nexport * from './dropdown-enumerator.js';\nexport * from './crud-interaction.js';\nexport * from './filter-pagination-crawler.js';\nexport * from './multi-context-handler.js';\nexport * from './advanced-ui-probes.js';\nexport * from './state-graph.js';\nexport * from './nested-window-crawler.js';\nexport * from './shadow-dom-crawler.js';\nexport * from './network-action-correlator.js';\nexport * from './crawl-session-manager.js';\nexport * from './accessibility-tree-crawler.js';\nexport * from './performance-metrics.js';\nexport * from './service-worker-interceptor.js';\n"],
+  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;AAE/B,SAAS,4BAA4B,8BAA8B,yBAAyB,8BAA8B;AAE1H,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;",
   "names": []
 }

package/dist/job-queue.js

@@ -222,11 +222,16 @@ async function createTrackedJob(type, config) {
   return job.id;
 }
 async function finishTrackedJob(tenantId, jobId, error) {
+  const nextRetryAt = error ? new Date(Date.now() + 6e4) : void 0;
   await withTenant(
     tenantId,
     (tx) => tx.platformJob.update({
       where: { id: jobId },
-      data: { status: error ? "FAILED" : "DONE", finishedAt: /* @__PURE__ */ new Date(), ...error ? { error } : { progress: 100 } }
+      data: {
+        status: error ? "FAILED" : "DONE",
+        finishedAt: /* @__PURE__ */ new Date(),
+        ...error ? { error, nextRetryAt } : { progress: 100 }
+      }
     })
   ).catch(() => {
   });
@@ -291,7 +296,7 @@ async function cancelJob(tenantId, jobId) {
     tenantId,
     (tx) => tx.platformJob.update({
       where: { id: jobId },
-      data: { status: "CANCELLED", finishedAt: /* @__PURE__ */ new Date() }
+      data: { status: "CANCELLED", finishedAt: /* @__PURE__ */ new Date(), heartbeatAt: null, workerId: null }
     })
   );
 }
@@ -425,7 +430,7 @@ async function processJob(jobId, config, opts) {
     await updateJobProgress(tenantId, jobId, pct, step);
     await opts?.onProgress?.(pct);
   };
-  const DEFAULT_MAX_SCREENS = 150;
+  const DEFAULT_MAX_SCREENS = parseInt(process.env.CRAWL_DEFAULT_MAX_SCREENS ?? "") || 150;
   try {
     if (!config.maxScreens || !config.deviceProfiles || config.interactive === void 0 || config.useSitemap === void 0 || !config.traversalStrategy) {
       const proj = await withTenant(
@@ -1308,7 +1313,7 @@ async function processDataRetentionPurge() {
       orderBy: { dataRetentionDays: "asc" },
       select: { dataRetentionDays: true }
     });
-    const retentionDays = setting?.dataRetentionDays ?? 365;
+    const retentionDays = setting?.dataRetentionDays ?? (parseInt(process.env.DEFAULT_DATA_RETENTION_DAYS ?? "") || 365);
     const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1e3);
     const result = await prismaAdmin.auditLog.deleteMany({
       where: { tenantId, createdAt: { lt: cutoff } }
@@ -1366,7 +1371,9 @@ async function processDiscoverJob(jobId, config, opts) {
     const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null) ?? {};
     const appUrl = config.appUrl ?? crawlCfg.appUrl;
     if (!appUrl) throw new Error("No app URL configured. Save the crawl settings first.");
-    const maxDiscoverUrls = config.maxDiscoverUrls ?? 2e3;
+    const { getSystemConfig } = await import("@detiq/core");
+    const crawlTuning = await getSystemConfig("crawl_tuning");
+    const maxDiscoverUrls = config.maxDiscoverUrls ?? parseInt(String(crawlTuning?.maxDiscoverUrls ?? "2000"), 10);
     const proxyUrl = config.proxyUrl ?? crawlCfg.proxyUrl;
     const screenshotDir = os.tmpdir();
     const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);