@detiq/crawler 1.1.13 → 1.1.15

package/dist/sitemap.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/sitemap.ts"],
-  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {\n    const subResults = await Promise.all(\n      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))\n    );\n    for (const sub of subResults) collected.push(...sub);\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
-  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,MAAI,eAAe,SAAS,KAAK,UAAU,SAAS,UAAU;AAC5D,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,eAAe,MAAM,GAAG,EAAE,EAAE,IAAI,YAAU,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAC;AAAA,IACnG;AACA,eAAW,OAAO,WAAY,WAAU,KAAK,GAAG,GAAG;AAAA,EACrD;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
+  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\n\nasync function getSitemapMaxUrls(): Promise<number> {\n  const { getSystemConfig } = await import('@detiq/core');\n  const cfg = await getSystemConfig('crawl_tuning');\n  return parseInt(String(cfg?.sitemapMaxUrls ?? '5000'), 10);\n}\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls = 5000): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any, maxUrls?: number): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  if (maxUrls === undefined) maxUrls = await getSitemapMaxUrls();\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);\n  const collected = [...pageUrls];\n\n  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {\n    const subResults = await Promise.all(\n      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))\n    );\n    for (const sub of subResults) collected.push(...sub);\n  }\n\n  return [...new Set(collected)].slice(0, maxUrls);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n  const maxUrls = await getSitemapMaxUrls();\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };\n}\n"],
+  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AAEzB,eAAe,oBAAqC;AAClD,QAAM,EAAE,gBAAgB,IAAI,MAAM,OAAO,aAAa;AACtD,QAAM,MAAM,MAAM,gBAAgB,cAAc;AAChD,SAAO,SAAS,OAAO,KAAK,kBAAkB,MAAM,GAAG,EAAE;AAC3D;AAEA,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAgB,UAAU,KAAwD;AAC1I,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,eAAe;AAChF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAkB,SAAqC;AAC3I,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,MAAI,YAAY,OAAW,WAAU,MAAM,kBAAkB;AAC7D,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,QAAQ,OAAO;AACzF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,MAAI,eAAe,SAAS,KAAK,UAAU,SAAS,SAAS;AAC3D,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,eAAe,MAAM,GAAG,EAAE,EAAE,IAAI,YAAU,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,YAAY,OAAO,CAAC;AAAA,IAC5G;AACA,eAAW,OAAO,WAAY,WAAU,KAAK,GAAG,GAAG;AAAA,EACrD;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO;AACjD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AACtD,QAAM,UAAU,MAAM,kBAAkB;AAGxC,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,YAAY,OAAO;AACtE,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,aAAa;AAC1E;",
   "names": []
 }

package/dist/state-graph.js

@@ -0,0 +1,193 @@
+function fnv1a32(token) {
+  let h = 2166136261;
+  for (let i = 0; i < token.length; i++) {
+    h = Math.imul(h ^ token.charCodeAt(i), 16777619);
+    h = h >>> 0;
+  }
+  return h;
+}
+function simHash32(text) {
+  const tokens = text.split(/\W+/).filter((t) => t.length > 2).slice(0, 500);
+  const vec = new Array(32).fill(0);
+  for (const token of tokens) {
+    const h = fnv1a32(token);
+    for (let b = 0; b < 32; b++) {
+      vec[b] += h >>> b & 1 ? 1 : -1;
+    }
+  }
+  let result = 0;
+  for (let b = 0; b < 32; b++) {
+    if (vec[b] > 0) result |= 1 << b;
+  }
+  return result >>> 0;
+}
+function hammingDistance32(a, b) {
+  let n = (a ^ b) >>> 0;
+  n = n - (n >>> 1 & 1431655765);
+  n = (n & 858993459) + (n >>> 2 & 858993459);
+  n = n + (n >>> 4) & 252645135;
+  return Math.imul(n, 16843009) >>> 24;
+}
+const STRIP_PARAMS = /* @__PURE__ */ new Set([
+  "utm_source",
+  "utm_medium",
+  "utm_campaign",
+  "ref",
+  "_ga",
+  "fbclid",
+  "session_id"
+]);
+function normalizeUrl(raw) {
+  try {
+    const u = new URL(raw);
+    u.hash = "";
+    const keep = [];
+    u.searchParams.forEach((v, k) => {
+      if (!STRIP_PARAMS.has(k)) keep.push([k, v]);
+    });
+    keep.sort((a, b) => a[0].localeCompare(b[0]));
+    u.search = keep.length ? "?" + keep.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&") : "";
+    return u.toString();
+  } catch {
+    return raw;
+  }
+}
+function computeStateId(normalizedUrl, hash) {
+  let path = normalizedUrl.slice(0, 40).replace(/[^a-zA-Z0-9]/g, "_");
+  const hex = hash.toString(16).padStart(8, "0");
+  return `${path}_${hex}`;
+}
+async function fingerprintPage(page, visitedAt) {
+  let visibleText = "";
+  let interactiveCount = 0;
+  try {
+    visibleText = await page.evaluate(() => {
+      const el = document.querySelector("main") ?? document.body;
+      return el.innerText ?? "";
+    });
+  } catch {
+    visibleText = "";
+  }
+  try {
+    interactiveCount = await page.evaluate(
+      () => document.querySelectorAll(
+        'a[href], button, input, select, textarea, [role="button"], [role="link"], [tabindex]'
+      ).length
+    );
+  } catch {
+    interactiveCount = 0;
+  }
+  const simHashValue = simHash32(visibleText + visitedAt.slice(0, 10));
+  const fingerprint = simHashValue.toString(16).padStart(8, "0");
+  return { fingerprint, visibleText, interactiveCount, simHashValue };
+}
+class ScreenTransitionGraph {
+  states = /* @__PURE__ */ new Map();
+  transitions = [];
+  /** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
+  hashIndex = /* @__PURE__ */ new Map();
+  // -------------------------------------------------------------------------
+  // State management
+  // -------------------------------------------------------------------------
+  addState(node) {
+    if (this.states.has(node.id)) {
+      return { added: false, duplicateOf: node.id };
+    }
+    for (const [indexedHash, ids] of this.hashIndex) {
+      if (hammingDistance32(node.simHashValue, indexedHash) < 3) {
+        return { added: false, duplicateOf: ids[0] };
+      }
+    }
+    this.states.set(node.id, node);
+    const bucket = this.hashIndex.get(node.simHashValue);
+    if (bucket) {
+      bucket.push(node.id);
+    } else {
+      this.hashIndex.set(node.simHashValue, [node.id]);
+    }
+    return { added: true };
+  }
+  addTransition(fromId, toId, action, label, triggeredAt) {
+    this.transitions.push({ fromId, toId, action, label, triggeredAt });
+  }
+  isDuplicate(url, visibleText, threshold = 3) {
+    const normalized = normalizeUrl(url);
+    const hash = simHash32(visibleText);
+    for (const [indexedHash] of this.hashIndex) {
+      if (hammingDistance32(hash, indexedHash) < threshold) return true;
+    }
+    for (const node of this.states.values()) {
+      if (node.normalizedUrl === normalized) return true;
+    }
+    return false;
+  }
+  // -------------------------------------------------------------------------
+  // Reporting
+  // -------------------------------------------------------------------------
+  getCoverageReport() {
+    const totalStates = this.states.size;
+    const totalTransitions = this.transitions.length;
+    const uniqueUrls = new Set(
+      Array.from(this.states.values()).map((n) => n.normalizedUrl)
+    ).size;
+    const interactiveElementsTotal = Array.from(this.states.values()).reduce(
+      (sum, n) => sum + n.interactiveElementCount,
+      0
+    );
+    const estimatedCoveragePercent = Math.min(
+      100,
+      totalTransitions / Math.max(1, totalStates) * 50
+    );
+    return {
+      totalStates,
+      totalTransitions,
+      uniqueUrls,
+      interactiveElementsTotal,
+      estimatedCoveragePercent
+    };
+  }
+  // -------------------------------------------------------------------------
+  // Serialisation
+  // -------------------------------------------------------------------------
+  toJSON() {
+    return {
+      states: Array.from(this.states.values()),
+      transitions: this.transitions
+    };
+  }
+  static fromJSON(data) {
+    const graph = new ScreenTransitionGraph();
+    for (const node of data.states) {
+      graph.states.set(node.id, node);
+      const bucket = graph.hashIndex.get(node.simHashValue);
+      if (bucket) {
+        bucket.push(node.id);
+      } else {
+        graph.hashIndex.set(node.simHashValue, [node.id]);
+      }
+    }
+    graph.transitions = [...data.transitions];
+    return graph;
+  }
+  // -------------------------------------------------------------------------
+  // Accessors
+  // -------------------------------------------------------------------------
+  getState(id) {
+    return this.states.get(id);
+  }
+  getAllStates() {
+    return Array.from(this.states.values());
+  }
+  getTransitionsFrom(id) {
+    return this.transitions.filter((t) => t.fromId === id);
+  }
+}
+export {
+  ScreenTransitionGraph,
+  computeStateId,
+  fingerprintPage,
+  hammingDistance32,
+  normalizeUrl,
+  simHash32
+};
+//# sourceMappingURL=state-graph.js.map

package/dist/state-graph.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/state-graph.ts"],
+  "sourcesContent": ["import type { Page } from 'playwright';\n\n// ---------------------------------------------------------------------------\n// Internal: FNV-1a 32-bit SimHash\n// ---------------------------------------------------------------------------\n\nfunction fnv1a32(token: string): number {\n  let h = 2166136261;\n  for (let i = 0; i < token.length; i++) {\n    h = Math.imul(h ^ token.charCodeAt(i), 16777619);\n    h = h >>> 0;\n  }\n  return h;\n}\n\nexport function simHash32(text: string): number {\n  const tokens = text\n    .split(/\\W+/)\n    .filter((t) => t.length > 2)\n    .slice(0, 500);\n\n  const vec = new Array<number>(32).fill(0);\n\n  for (const token of tokens) {\n    const h = fnv1a32(token);\n    for (let b = 0; b < 32; b++) {\n      vec[b] += (h >>> b) & 1 ? 1 : -1;\n    }\n  }\n\n  let result = 0;\n  for (let b = 0; b < 32; b++) {\n    if (vec[b] > 0) result |= 1 << b;\n  }\n  return result >>> 0;\n}\n\nexport function hammingDistance32(a: number, b: number): number {\n  let n = (a ^ b) >>> 0;\n  n = n - ((n >>> 1) & 0x55555555);\n  n = (n & 0x33333333) + ((n >>> 2) & 0x33333333);\n  n = (n + (n >>> 4)) & 0x0f0f0f0f;\n  return Math.imul(n, 0x01010101) >>> 24;\n}\n\n// ---------------------------------------------------------------------------\n// URL normalisation\n// ---------------------------------------------------------------------------\n\nconst STRIP_PARAMS = new Set([\n  'utm_source', 'utm_medium', 'utm_campaign',\n  'ref', '_ga', 'fbclid', 'session_id',\n]);\n\nexport function normalizeUrl(raw: string): string {\n  try {\n    const u = new URL(raw);\n    u.hash = '';\n    const keep: Array<[string, string]> = [];\n    u.searchParams.forEach((v, k) => {\n      if (!STRIP_PARAMS.has(k)) keep.push([k, v]);\n    });\n    keep.sort((a, b) => a[0].localeCompare(b[0]));\n    u.search = keep.length\n      ? '?' + keep.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')\n      : '';\n    return u.toString();\n  } catch {\n    return raw;\n  }\n}\n\nexport function computeStateId(normalizedUrl: string, hash: number): string {\n  let path = normalizedUrl.slice(0, 40).replace(/[^a-zA-Z0-9]/g, '_');\n  const hex = hash.toString(16).padStart(8, '0');\n  return `${path}_${hex}`;\n}\n\n// ---------------------------------------------------------------------------\n// Public types\n// ---------------------------------------------------------------------------\n\nexport interface UIStateNode {\n  id: string;\n  url: string;\n  normalizedUrl: string;\n  title: string;\n  interactiveElementCount: number;\n  domFingerprint: string;\n  simHashValue: number;\n  visitedAt: string;\n}\n\nexport interface StateTransition {\n  fromId: string;\n  toId: string;\n  action: string;\n  label: string;\n  triggeredAt: string;\n}\n\nexport interface CoverageReport {\n  totalStates: number;\n  totalTransitions: number;\n  uniqueUrls: number;\n  interactiveElementsTotal: number;\n  estimatedCoveragePercent: number;\n}\n\n// ---------------------------------------------------------------------------\n// Page fingerprinting\n// ---------------------------------------------------------------------------\n\nexport async function fingerprintPage(\n  page: Page,\n  visitedAt: string,\n): Promise<{\n  fingerprint: string;\n  visibleText: string;\n  interactiveCount: number;\n  simHashValue: number;\n}> {\n  let visibleText = '';\n  let interactiveCount = 0;\n\n  try {\n    visibleText = await page.evaluate(() => {\n      const el = document.querySelector('main') ?? document.body;\n      return (el as HTMLElement).innerText ?? '';\n    });\n  } catch {\n    visibleText = '';\n  }\n\n  try {\n    interactiveCount = await page.evaluate(() =>\n      document.querySelectorAll(\n        'a[href], button, input, select, textarea, [role=\"button\"], [role=\"link\"], [tabindex]',\n      ).length,\n    );\n  } catch {\n    interactiveCount = 0;\n  }\n\n  const simHashValue = simHash32(visibleText + visitedAt.slice(0, 10));\n  const fingerprint = simHashValue.toString(16).padStart(8, '0');\n\n  return { fingerprint, visibleText, interactiveCount, simHashValue };\n}\n\n// ---------------------------------------------------------------------------\n// ScreenTransitionGraph\n// ---------------------------------------------------------------------------\n\nexport class ScreenTransitionGraph {\n  private states = new Map<string, UIStateNode>();\n  private transitions: StateTransition[] = [];\n  /** Maps simHash value \u2192 array of state ids with that hash (for near-dup lookup) */\n  private hashIndex = new Map<number, string[]>();\n\n  // -------------------------------------------------------------------------\n  // State management\n  // -------------------------------------------------------------------------\n\n  addState(node: UIStateNode): { added: boolean; duplicateOf?: string } {\n    if (this.states.has(node.id)) {\n      return { added: false, duplicateOf: node.id };\n    }\n\n    // Check near-duplicate via hamming distance across all indexed hashes\n    for (const [indexedHash, ids] of this.hashIndex) {\n      if (hammingDistance32(node.simHashValue, indexedHash) < 3) {\n        return { added: false, duplicateOf: ids[0] };\n      }\n    }\n\n    this.states.set(node.id, node);\n\n    const bucket = this.hashIndex.get(node.simHashValue);\n    if (bucket) {\n      bucket.push(node.id);\n    } else {\n      this.hashIndex.set(node.simHashValue, [node.id]);\n    }\n\n    return { added: true };\n  }\n\n  addTransition(\n    fromId: string,\n    toId: string,\n    action: string,\n    label: string,\n    triggeredAt: string,\n  ): void {\n    this.transitions.push({ fromId, toId, action, label, triggeredAt });\n  }\n\n  isDuplicate(url: string, visibleText: string, threshold = 3): boolean {\n    const normalized = normalizeUrl(url);\n    const hash = simHash32(visibleText);\n    for (const [indexedHash] of this.hashIndex) {\n      if (hammingDistance32(hash, indexedHash) < threshold) return true;\n    }\n    // Also check exact normalized URL match\n    for (const node of this.states.values()) {\n      if (node.normalizedUrl === normalized) return true;\n    }\n    return false;\n  }\n\n  // -------------------------------------------------------------------------\n  // Reporting\n  // -------------------------------------------------------------------------\n\n  getCoverageReport(): CoverageReport {\n    const totalStates = this.states.size;\n    const totalTransitions = this.transitions.length;\n    const uniqueUrls = new Set(\n      Array.from(this.states.values()).map((n) => n.normalizedUrl),\n    ).size;\n    const interactiveElementsTotal = Array.from(this.states.values()).reduce(\n      (sum, n) => sum + n.interactiveElementCount,\n      0,\n    );\n    const estimatedCoveragePercent = Math.min(\n      100,\n      (totalTransitions / Math.max(1, totalStates)) * 50,\n    );\n    return {\n      totalStates,\n      totalTransitions,\n      uniqueUrls,\n      interactiveElementsTotal,\n      estimatedCoveragePercent,\n    };\n  }\n\n  // -------------------------------------------------------------------------\n  // Serialisation\n  // -------------------------------------------------------------------------\n\n  toJSON(): { states: UIStateNode[]; transitions: StateTransition[] } {\n    return {\n      states: Array.from(this.states.values()),\n      transitions: this.transitions,\n    };\n  }\n\n  static fromJSON(data: { states: UIStateNode[]; transitions: StateTransition[] }): ScreenTransitionGraph {\n    const graph = new ScreenTransitionGraph();\n    for (const node of data.states) {\n      graph.states.set(node.id, node);\n      const bucket = graph.hashIndex.get(node.simHashValue);\n      if (bucket) {\n        bucket.push(node.id);\n      } else {\n        graph.hashIndex.set(node.simHashValue, [node.id]);\n      }\n    }\n    graph.transitions = [...data.transitions];\n    return graph;\n  }\n\n  // -------------------------------------------------------------------------\n  // Accessors\n  // -------------------------------------------------------------------------\n\n  getState(id: string): UIStateNode | undefined {\n    return this.states.get(id);\n  }\n\n  getAllStates(): UIStateNode[] {\n    return Array.from(this.states.values());\n  }\n\n  getTransitionsFrom(id: string): StateTransition[] {\n    return this.transitions.filter((t) => t.fromId === id);\n  }\n}\n"],
+  "mappings": "AAMA,SAAS,QAAQ,OAAuB;AACtC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,QAAI,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC,GAAG,QAAQ;AAC/C,QAAI,MAAM;AAAA,EACZ;AACA,SAAO;AACT;AAEO,SAAS,UAAU,MAAsB;AAC9C,QAAM,SAAS,KACZ,MAAM,KAAK,EACX,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAC1B,MAAM,GAAG,GAAG;AAEf,QAAM,MAAM,IAAI,MAAc,EAAE,EAAE,KAAK,CAAC;AAExC,aAAW,SAAS,QAAQ;AAC1B,UAAM,IAAI,QAAQ,KAAK;AACvB,aAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC3B,UAAI,CAAC,KAAM,MAAM,IAAK,IAAI,IAAI;AAAA,IAChC;AAAA,EACF;AAEA,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC3B,QAAI,IAAI,CAAC,IAAI,EAAG,WAAU,KAAK;AAAA,EACjC;AACA,SAAO,WAAW;AACpB;AAEO,SAAS,kBAAkB,GAAW,GAAmB;AAC9D,MAAI,KAAK,IAAI,OAAO;AACpB,MAAI,KAAM,MAAM,IAAK;AACrB,OAAK,IAAI,cAAgB,MAAM,IAAK;AACpC,MAAK,KAAK,MAAM,KAAM;AACtB,SAAO,KAAK,KAAK,GAAG,QAAU,MAAM;AACtC;AAMA,MAAM,eAAe,oBAAI,IAAI;AAAA,EAC3B;AAAA,EAAc;AAAA,EAAc;AAAA,EAC5B;AAAA,EAAO;AAAA,EAAO;AAAA,EAAU;AAC1B,CAAC;AAEM,SAAS,aAAa,KAAqB;AAChD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,MAAE,OAAO;AACT,UAAM,OAAgC,CAAC;AACvC,MAAE,aAAa,QAAQ,CAAC,GAAG,MAAM;AAC/B,UAAI,CAAC,aAAa,IAAI,CAAC,EAAG,MAAK,KAAK,CAAC,GAAG,CAAC,CAAC;AAAA,IAC5C,CAAC;AACD,SAAK,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;AAC5C,MAAE,SAAS,KAAK,SACZ,MAAM,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,mBAAmB,CAAC,CAAC,IAAI,mBAAmB,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,IACxF;AACJ,WAAO,EAAE,SAAS;AAAA,EACpB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,eAAe,eAAuB,MAAsB;AAC1E,MAAI,OAAO,cAAc,MAAM,GAAG,EAAE,EAAE,QAAQ,iBAAiB,GAAG;AAClE,QAAM,MAAM,KAAK,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAC7C,SAAO,GAAG,IAAI,IAAI,GAAG;AACvB;AAqCA,eAAsB,gBACpB,MACA,WAMC;AACD,MAAI,cAAc;AAClB,MAAI,mBAAmB;AAEvB,MAAI;AACF,kBAAc,MAAM,KAAK,SAAS,MAAM;AACtC,YAAM,KAAK,SAAS,cAAc,MAAM,KAAK,SAAS;AACtD,aAAQ,GAAmB,aAAa;AAAA,IAC1C,CAAC;AAAA,EACH,QAAQ;AACN,kBAAc;AAAA,EAChB;AAEA,MAAI;AACF,uBAAmB,MAAM,KAAK;AAAA,MAAS,MACrC,SAAS;AAAA,QACP;AAAA,MACF,EAAE;AAAA,IACJ;AAAA,EACF,QAAQ;AACN,uBAAmB;AAAA,EACrB;AAEA,QAAM,eAAe,UAAU,cAAc,UAAU,MAAM,GAAG,EAAE,CAAC;AACnE,QAAM,cAAc,aAAa,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAE7D,SAAO,EAAE,aAAa,aAAa,kBAAkB,aAAa;AACpE;AAMO,MAAM,sBAAsB;AAAA,EACzB,SAAS,oBAAI,IAAyB;AAAA,EACtC,cAAiC,CAAC;AAAA;AAAA,EAElC,YAAY,oBAAI,IAAsB;AAAA;AAAA;AAAA;AAAA,EAM9C,SAAS,MAA6D;AACpE,QAAI,KAAK,OAAO,IAAI,KAAK,EAAE,GAAG;AAC5B,aAAO,EAAE,OAAO,OAAO,aAAa,KAAK,GAAG;AAAA,IAC9C;AAGA,eAAW,CAAC,aAAa,GAAG,KAAK,KAAK,WAAW;AAC/C,UAAI,kBAAkB,KAAK,cAAc,WAAW,IAAI,GAAG;AACzD,eAAO,EAAE,OAAO,OAAO,aAAa,IAAI,CAAC,EAAE;AAAA,MAC7C;AAAA,IACF;AAEA,SAAK,OAAO,IAAI,KAAK,IAAI,IAAI;AAE7B,UAAM,SAAS,KAAK,UAAU,IAAI,KAAK,YAAY;AACnD,QAAI,QAAQ;AACV,aAAO,KAAK,KAAK,EAAE;AAAA,IACrB,OAAO;AACL,WAAK,UAAU,IAAI,KAAK,cAAc,CAAC,KAAK,EAAE,CAAC;AAAA,IACjD;AAEA,WAAO,EAAE,OAAO,KAAK;AAAA,EACvB;AAAA,EAEA,cACE,QACA,MACA,QACA,OACA,aACM;AACN,SAAK,YAAY,KAAK,EAAE,QAAQ,MAAM,QAAQ,OAAO,YAAY,CAAC;AAAA,EACpE;AAAA,EAEA,YAAY,KAAa,aAAqB,YAAY,GAAY;AACpE,UAAM,aAAa,aAAa,GAAG;AACnC,UAAM,OAAO,UAAU,WAAW;AAClC,eAAW,CAAC,WAAW,KAAK,KAAK,WAAW;AAC1C,UAAI,kBAAkB,MAAM,WAAW,IAAI,UAAW,QAAO;AAAA,IAC/D;AAEA,eAAW,QAAQ,KAAK,OAAO,OAAO,GAAG;AACvC,UAAI,KAAK,kBAAkB,WAAY,QAAO;AAAA,IAChD;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,oBAAoC;AAClC,UAAM,cAAc,KAAK,OAAO;AAChC,UAAM,mBAAmB,KAAK,YAAY;AAC1C,UAAM,aAAa,IAAI;AAAA,MACrB,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa;AAAA,IAC7D,EAAE;AACF,UAAM,2BAA2B,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC,EAAE;AAAA,MAChE,CAAC,KAAK,MAAM,MAAM,EAAE;AAAA,MACpB;AAAA,IACF;AACA,UAAM,2BAA2B,KAAK;AAAA,MACpC;AAAA,MACC,mBAAmB,KAAK,IAAI,GAAG,WAAW,IAAK;AAAA,IAClD;AACA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,SAAoE;AAClE,WAAO;AAAA,MACL,QAAQ,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC;AAAA,MACvC,aAAa,KAAK;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,OAAO,SAAS,MAAwF;AACtG,UAAM,QAAQ,IAAI,sBAAsB;AACxC,eAAW,QAAQ,KAAK,QAAQ;AAC9B,YAAM,OAAO,IAAI,KAAK,IAAI,IAAI;AAC9B,YAAM,SAAS,MAAM,UAAU,IAAI,KAAK,YAAY;AACpD,UAAI,QAAQ;AACV,eAAO,KAAK,KAAK,EAAE;AAAA,MACrB,OAAO;AACL,cAAM,UAAU,IAAI,KAAK,cAAc,CAAC,KAAK,EAAE,CAAC;AAAA,MAClD;AAAA,IACF;AACA,UAAM,cAAc,CAAC,GAAG,KAAK,WAAW;AACxC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,IAAqC;AAC5C,WAAO,KAAK,OAAO,IAAI,EAAE;AAAA,EAC3B;AAAA,EAEA,eAA8B;AAC5B,WAAO,MAAM,KAAK,KAAK,OAAO,OAAO,CAAC;AAAA,EACxC;AAAA,EAEA,mBAAmB,IAA+B;AAChD,WAAO,KAAK,YAAY,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE;AAAA,EACvD;AACF;",
+  "names": []
+}

package/dist/stealth/human-behavior.js

@@ -86,20 +86,51 @@ async function injectFingerprintOverrides(page) {
   }).catch(() => {
   });
 }
-async function infiniteScrollToBottom(page, opts = {}) {
+async function infiniteScrollToBottom(page, opts = {}, rawPage) {
   const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
   let scrollCount = 0;
   let stableScrollCount = 0;
   let previousHeight = 0;
+  let cdpForWheel = null;
+  let viewW = 1280;
+  let viewH = 800;
+  if (rawPage && typeof rawPage.context === "function") {
+    try {
+      const ctx = rawPage.context();
+      if (typeof ctx?.newCDPSession === "function") {
+        cdpForWheel = await ctx.newCDPSession(rawPage);
+        const { result } = await cdpForWheel.send("Runtime.evaluate", {
+          expression: `({"w":window.innerWidth,"h":window.innerHeight})`,
+          returnByValue: true
+        }).catch(() => ({ result: { value: null } }));
+        if (result?.value) {
+          viewW = result.value.w ?? 1280;
+          viewH = result.value.h ?? 800;
+        }
+      }
+    } catch {
+      cdpForWheel = null;
+    }
+  }
   try {
     while (scrollCount < maxScrolls) {
       previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
       const scrollBy = 500 + Math.floor(Math.random() * 300);
       await page.evaluate((by) => window.scrollBy(0, by), scrollBy).catch(() => {
       });
+      if (cdpForWheel) {
+        await cdpForWheel.send("Input.dispatchMouseEvent", {
+          type: "mouseWheel",
+          x: Math.floor(viewW / 2),
+          y: Math.floor(viewH / 2),
+          deltaX: 0,
+          deltaY: scrollBy,
+          modifiers: 0
+        }).catch(() => {
+        });
+      }
       await humanDelay(scrollDelay - 100, scrollDelay + 200);
-      await page.waitForTimeout(stabilizeMs).catch(() => {
-      });
+      await new Promise((r) => setTimeout(r, stabilizeMs));
       scrollCount++;
       const newHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
       if (newHeight - previousHeight < 10) {
@@ -112,6 +143,8 @@ async function infiniteScrollToBottom(page, opts = {}) {
     }
   } catch {
   }
+  if (cdpForWheel) cdpForWheel.detach().catch(() => {
+  });
   return { scrollCount, heightReached: previousHeight };
 }
 async function extractIframeContent(page) {

package/dist/stealth/human-behavior.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/stealth/human-behavior.ts"],
-  "sourcesContent": ["import type { Page } from 'playwright';\n\n// Gaussian random using Box-Muller transform\nfunction gaussian(mean: number, sigma: number): number {\n  const u1 = Math.random();\n  const u2 = Math.random();\n  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);\n  return mean + z * sigma;\n}\n\nexport async function humanDelay(min = 80, max = 450): Promise<void> {\n  const mean = (min + max) / 2;\n  const sigma = (max - min) / 6;\n  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));\n  await new Promise(r => setTimeout(r, Math.round(delay)));\n}\n\nexport async function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void> {\n  try {\n    const mouse = page.mouse;\n    const steps = 8 + Math.floor(Math.random() * 8);\n    // Generate Bezier control points with slight perpendicular offset\n    const startX = targetX - 100 + Math.random() * 200;\n    const startY = targetY - 100 + Math.random() * 200;\n    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;\n    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;\n\n    for (let i = 0; i <= steps; i++) {\n      const t = i / steps;\n      // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2\n      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;\n      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;\n      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);\n      if (i < steps) await humanDelay(8, 25);\n    }\n    // Small overshoot correction\n    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);\n    await humanDelay(30, 80);\n    await mouse.move(targetX, targetY);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateMouseHover(page: Page, selector: string): Promise<void> {\n  try {\n    const el = await page.$(selector);\n    if (!el) return;\n    const box = await el.boundingBox();\n    if (!box) return;\n    const cx = box.x + box.width / 2;\n    const cy = box.y + box.height / 2;\n    await simulateMouseMovement(page, cx, cy);\n    await humanDelay(100, 300);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateClick(page: Page, selector: string): Promise<void> {\n  try {\n    await simulateMouseHover(page, selector);\n    await humanDelay(50, 150);\n    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });\n  } catch { /* non-fatal */ }\n}\n\nexport async function injectFingerprintOverrides(page: Page): Promise<void> {\n  await page.addInitScript(() => {\n    // Canvas noise \u2014 subtle per-pixel perturbation\n    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;\n    HTMLCanvasElement.prototype.toDataURL = function(this: HTMLCanvasElement, ...args: any[]) {\n      const ctx = this.getContext('2d');\n      if (ctx) {\n        const imageData = ctx.getImageData(0, 0, this.width, this.height);\n        for (let i = 0; i < imageData.data.length; i += 4) {\n          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));\n        }\n        ctx.putImageData(imageData, 0, 0);\n      }\n      return origToDataURL.apply(this, args as [type?: string, quality?: number]);\n    };\n\n    // WebGL parameter spoofing\n    const origGetParam = WebGLRenderingContext.prototype.getParameter;\n    WebGLRenderingContext.prototype.getParameter = function(this: WebGLRenderingContext, parameter: number) {\n      const UNMASKED_VENDOR_WEBGL = 0x9245;\n      const UNMASKED_RENDERER_WEBGL = 0x9246;\n      if (parameter === UNMASKED_VENDOR_WEBGL) return 'Intel Inc.';\n      if (parameter === UNMASKED_RENDERER_WEBGL) return 'Intel Iris OpenGL Engine';\n      return origGetParam.call(this, parameter);\n    };\n\n    // Hardware concurrency \u2014 pick from common values\n    const cpuCounts = [4, 6, 8, 8];\n    Object.defineProperty(navigator, 'hardwareConcurrency', {\n      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],\n      configurable: true,\n    });\n\n    // Device memory\n    Object.defineProperty(navigator, 'deviceMemory', {\n      get: () => [4, 8][Math.floor(Math.random() * 2)],\n      configurable: true,\n    });\n  }).catch(() => {});\n}\n\nexport async function infiniteScrollToBottom(\n  page: Page,\n  opts: { maxScrolls?: number; scrollDelay?: number; stabilizeMs?: number } = {}\n): Promise<{ scrollCount: number; heightReached: number }> {\n  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;\n  let scrollCount = 0;\n  let stableScrollCount = 0;\n  let previousHeight = 0;\n\n  try {\n    while (scrollCount < maxScrolls) {\n      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);\n\n      const scrollBy = 500 + Math.floor(Math.random() * 300);\n      await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});\n      await humanDelay(scrollDelay - 100, scrollDelay + 200);\n\n      // Wait for potential lazy-loaded content\n      await page.waitForTimeout(stabilizeMs).catch(() => {});\n      scrollCount++;\n\n      const newHeight: number = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);\n      if (newHeight - previousHeight < 10) {\n        stableScrollCount++;\n        if (stableScrollCount >= 3) break; // no meaningful new content after 3 consecutive non-growing scrolls\n      } else {\n        stableScrollCount = 0;\n      }\n      previousHeight = newHeight;\n    }\n  } catch { /* non-fatal */ }\n\n  return { scrollCount, heightReached: previousHeight };\n}\n\nexport async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {\n  const results: Array<{ src: string; content: string; markdown: string }> = [];\n  try {\n    const iframeSrcs: string[] = await page.evaluate(() =>\n      Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')\n    ).catch(() => []);\n\n    for (const frame of page.frames()) {\n      const src = frame.url();\n      if (!src || src === 'about:blank' || src === page.url()) continue;\n      try {\n        const content = await frame.content().catch(() => '');\n        if (!content || content.length < 50) continue;\n        // Simple text extraction from iframe HTML\n        const text = content.replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n          .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n          .replace(/<[^>]+>/g, ' ')\n          .replace(/\\s+/g, ' ')\n          .trim();\n        results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });\n      } catch { /* skip this frame */ }\n    }\n  } catch { /* non-fatal */ }\n  return results;\n}\n\nexport async function pierceShadowDom(page: Page): Promise<string> {\n  try {\n    return await page.evaluate(() => {\n      const parts: string[] = [];\n      function walkShadow(root: Element | ShadowRoot): void {\n        const shadow = (root as any).shadowRoot;\n        if (shadow) {\n          parts.push(shadow.innerHTML ?? '');\n          for (const child of [...shadow.querySelectorAll('*')]) walkShadow(child);\n        }\n        for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {\n          if ((child as any).shadowRoot) walkShadow(child);\n        }\n      }\n      walkShadow(document.body);\n      return parts\n        .join('\\n')\n        .replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n        .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n        .replace(/<[^>]+>/g, ' ')\n        .replace(/\\s+/g, ' ')\n        .trim()\n        .slice(0, 10_000);\n    }).catch(() => '');\n  } catch {\n    return '';\n  }\n}\n"],
-  "mappings": "AAGA,SAAS,SAAS,MAAc,OAAuB;AACrD,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,IAAI,KAAK,KAAK,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;AAClE,SAAO,OAAO,IAAI;AACpB;AAEA,eAAsB,WAAW,MAAM,IAAI,MAAM,KAAoB;AACnE,QAAM,QAAQ,MAAM,OAAO;AAC3B,QAAM,SAAS,MAAM,OAAO;AAC5B,QAAM,QAAQ,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,SAAS,MAAM,KAAK,CAAC,CAAC;AAChE,QAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,KAAK,MAAM,KAAK,CAAC,CAAC;AACzD;AAEA,eAAsB,sBAAsB,MAAY,SAAiB,SAAgC;AACvG,MAAI;AACF,UAAM,QAAQ,KAAK;AACnB,UAAM,QAAQ,IAAI,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC;AAE9C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAC7D,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAE7D,aAAS,IAAI,GAAG,KAAK,OAAO,KAAK;AAC/B,YAAM,IAAI,IAAI;AAEd,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,MAAM,KAAK,KAAK,KAAK,OAAO,IAAI,OAAO,GAAG,KAAK,KAAK,OAAO,IAAI,OAAO,CAAC;AAC7E,UAAI,IAAI,MAAO,OAAM,WAAW,GAAG,EAAE;AAAA,IACvC;AAEA,UAAM,MAAM,KAAK,WAAW,KAAK,OAAO,IAAI,OAAO,GAAG,WAAW,KAAK,OAAO,IAAI,OAAO,CAAC;AACzF,UAAM,WAAW,IAAI,EAAE;AACvB,UAAM,MAAM,KAAK,SAAS,OAAO;AAAA,EACnC,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,mBAAmB,MAAY,UAAiC;AACpF,MAAI;AACF,UAAM,KAAK,MAAM,KAAK,EAAE,QAAQ;AAChC,QAAI,CAAC,GAAI;AACT,UAAM,MAAM,MAAM,GAAG,YAAY;AACjC,QAAI,CAAC,IAAK;AACV,UAAM,KAAK,IAAI,IAAI,IAAI,QAAQ;AAC/B,UAAM,KAAK,IAAI,IAAI,IAAI,SAAS;AAChC,UAAM,sBAAsB,MAAM,IAAI,EAAE;AACxC,UAAM,WAAW,KAAK,GAAG;AAAA,EAC3B,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,cAAc,MAAY,UAAiC;AAC/E,MAAI;AACF,UAAM,mBAAmB,MAAM,QAAQ;AACvC,UAAM,WAAW,IAAI,GAAG;AACxB,UAAM,KAAK,MAAM,UAAU,EAAE,OAAO,KAAK,MAAM,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;AAAA,EACpE,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,2BAA2B,MAA2B;AAC1E,QAAM,KAAK,cAAc,MAAM;AAE7B,UAAM,gBAAgB,kBAAkB,UAAU;AAClD,sBAAkB,UAAU,YAAY,YAAqC,MAAa;AACxF,YAAM,MAAM,KAAK,WAAW,IAAI;AAChC,UAAI,KAAK;AACP,cAAM,YAAY,IAAI,aAAa,GAAG,GAAG,KAAK,OAAO,KAAK,MAAM;AAChE,iBAAS,IAAI,GAAG,IAAI,UAAU,KAAK,QAAQ,KAAK,GAAG;AACjD,oBAAU,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,OAAO,IAAI,OAAQ,KAAK,OAAO,IAAI,MAAM,IAAI,KAAM,EAAE,CAAC;AAAA,QAChI;AACA,YAAI,aAAa,WAAW,GAAG,CAAC;AAAA,MAClC;AACA,aAAO,cAAc,MAAM,MAAM,IAAyC;AAAA,IAC5E;AAGA,UAAM,eAAe,sBAAsB,UAAU;AACrD,0BAAsB,UAAU,eAAe,SAAsC,WAAmB;AACtG,YAAM,wBAAwB;AAC9B,YAAM,0BAA0B;AAChC,UAAI,cAAc,sBAAuB,QAAO;AAChD,UAAI,cAAc,wBAAyB,QAAO;AAClD,aAAO,aAAa,KAAK,MAAM,SAAS;AAAA,IAC1C;AAGA,UAAM,YAAY,CAAC,GAAG,GAAG,GAAG,CAAC;AAC7B,WAAO,eAAe,WAAW,uBAAuB;AAAA,MACtD,KAAK,MAAM,UAAU,KAAK,MAAM,KAAK,OAAO,IAAI,UAAU,MAAM,CAAC;AAAA,MACjE,cAAc;AAAA,IAChB,CAAC;AAGD,WAAO,eAAe,WAAW,gBAAgB;AAAA,MAC/C,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC;AAAA,MAC/C,cAAc;AAAA,IAChB,CAAC;AAAA,EACH,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACnB;AAEA,eAAsB,uBACpB,MACA,OAA4E,CAAC,GACpB;AACzD,QAAM,EAAE,aAAa,IAAI,cAAc,KAAK,cAAc,IAAI,IAAI;AAClE,MAAI,cAAc;AAClB,MAAI,oBAAoB;AACxB,MAAI,iBAAiB;AAErB,MAAI;AACF,WAAO,cAAc,YAAY;AAC/B,uBAAiB,MAAM,KAAK,SAAS,MAAM,SAAS,gBAAgB,YAAY,EAAE,MAAM,MAAM,CAAC;AAE/F,YAAM,WAAW,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACrD,YAAM,KAAK,SAAS,CAAC,OAAe,OAAO,SAAS,GAAG,EAAE,GAAG,QAAQ,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACpF,YAAM,WAAW,cAAc,KAAK,cAAc,GAAG;AAGrD,YAAM,KAAK,eAAe,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACrD;AAEA,YAAM,YAAoB,MAAM,KAAK,SAAS,MAAM,SAAS,gBAAgB,YAAY,EAAE,MAAM,MAAM,CAAC;AACxG,UAAI,YAAY,iBAAiB,IAAI;AACnC;AACA,YAAI,qBAAqB,EAAG;AAAA,MAC9B,OAAO;AACL,4BAAoB;AAAA,MACtB;AACA,uBAAiB;AAAA,IACnB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAE1B,SAAO,EAAE,aAAa,eAAe,eAAe;AACtD;AAEA,eAAsB,qBAAqB,MAAgF;AACzH,QAAM,UAAqE,CAAC;AAC5E,MAAI;AACF,UAAM,aAAuB,MAAM,KAAK;AAAA,MAAS,MAC/C,MAAM,KAAK,SAAS,iBAAiB,QAAQ,CAAC,EAAE,IAAI,OAAK,EAAE,aAAa,KAAK,KAAK,EAAE,aAAa,UAAU,KAAK,EAAE;AAAA,IACpH,EAAE,MAAM,MAAM,CAAC,CAAC;AAEhB,eAAW,SAAS,KAAK,OAAO,GAAG;AACjC,YAAM,MAAM,MAAM,IAAI;AACtB,UAAI,CAAC,OAAO,QAAQ,iBAAiB,QAAQ,KAAK,IAAI,EAAG;AACzD,UAAI;AACF,cAAM,UAAU,MAAM,MAAM,QAAQ,EAAE,MAAM,MAAM,EAAE;AACpD,YAAI,CAAC,WAAW,QAAQ,SAAS,GAAI;AAErC,cAAM,OAAO,QAAQ,QAAQ,qCAAqC,EAAE,EACjE,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACR,gBAAQ,KAAK,EAAE,KAAK,SAAS,QAAQ,MAAM,GAAG,GAAM,GAAG,UAAU,KAAK,MAAM,GAAG,GAAK,EAAE,CAAC;AAAA,MACzF,QAAQ;AAAA,MAAwB;AAAA,IAClC;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,eAAsB,gBAAgB,MAA6B;AACjE,MAAI;AACF,WAAO,MAAM,KAAK,SAAS,MAAM;AAC/B,YAAM,QAAkB,CAAC;AACzB,eAAS,WAAW,MAAkC;AACpD,cAAM,SAAU,KAAa;AAC7B,YAAI,QAAQ;AACV,gBAAM,KAAK,OAAO,aAAa,EAAE;AACjC,qBAAW,SAAS,CAAC,GAAG,OAAO,iBAAiB,GAAG,CAAC,EAAG,YAAW,KAAK;AAAA,QACzE;AACA,mBAAW,SAAS,MAAM,KAAK,KAAK,iBAAiB,yBAAyB,CAAC,GAAG;AAChF,cAAK,MAAc,WAAY,YAAW,KAAK;AAAA,QACjD;AAAA,MACF;AACA,iBAAW,SAAS,IAAI;AACxB,aAAO,MACJ,KAAK,IAAI,EACT,QAAQ,qCAAqC,EAAE,EAC/C,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK,EACL,MAAM,GAAG,GAAM;AAAA,IACpB,CAAC,EAAE,MAAM,MAAM,EAAE;AAAA,EACnB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "sourcesContent": ["import type { Page } from 'playwright';\n\n// Gaussian random using Box-Muller transform\nfunction gaussian(mean: number, sigma: number): number {\n  const u1 = Math.random();\n  const u2 = Math.random();\n  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);\n  return mean + z * sigma;\n}\n\nexport async function humanDelay(min = 80, max = 450): Promise<void> {\n  const mean = (min + max) / 2;\n  const sigma = (max - min) / 6;\n  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));\n  await new Promise(r => setTimeout(r, Math.round(delay)));\n}\n\nexport async function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void> {\n  try {\n    const mouse = page.mouse;\n    const steps = 8 + Math.floor(Math.random() * 8);\n    // Generate Bezier control points with slight perpendicular offset\n    const startX = targetX - 100 + Math.random() * 200;\n    const startY = targetY - 100 + Math.random() * 200;\n    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;\n    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;\n\n    for (let i = 0; i <= steps; i++) {\n      const t = i / steps;\n      // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2\n      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;\n      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;\n      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);\n      if (i < steps) await humanDelay(8, 25);\n    }\n    // Small overshoot correction\n    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);\n    await humanDelay(30, 80);\n    await mouse.move(targetX, targetY);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateMouseHover(page: Page, selector: string): Promise<void> {\n  try {\n    const el = await page.$(selector);\n    if (!el) return;\n    const box = await el.boundingBox();\n    if (!box) return;\n    const cx = box.x + box.width / 2;\n    const cy = box.y + box.height / 2;\n    await simulateMouseMovement(page, cx, cy);\n    await humanDelay(100, 300);\n  } catch { /* non-fatal */ }\n}\n\nexport async function simulateClick(page: Page, selector: string): Promise<void> {\n  try {\n    await simulateMouseHover(page, selector);\n    await humanDelay(50, 150);\n    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });\n  } catch { /* non-fatal */ }\n}\n\nexport async function injectFingerprintOverrides(page: Page): Promise<void> {\n  await page.addInitScript(() => {\n    // Canvas noise \u2014 subtle per-pixel perturbation\n    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;\n    HTMLCanvasElement.prototype.toDataURL = function(this: HTMLCanvasElement, ...args: any[]) {\n      const ctx = this.getContext('2d');\n      if (ctx) {\n        const imageData = ctx.getImageData(0, 0, this.width, this.height);\n        for (let i = 0; i < imageData.data.length; i += 4) {\n          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));\n        }\n        ctx.putImageData(imageData, 0, 0);\n      }\n      return origToDataURL.apply(this, args as [type?: string, quality?: number]);\n    };\n\n    // WebGL parameter spoofing\n    const origGetParam = WebGLRenderingContext.prototype.getParameter;\n    WebGLRenderingContext.prototype.getParameter = function(this: WebGLRenderingContext, parameter: number) {\n      const UNMASKED_VENDOR_WEBGL = 0x9245;\n      const UNMASKED_RENDERER_WEBGL = 0x9246;\n      if (parameter === UNMASKED_VENDOR_WEBGL) return 'Intel Inc.';\n      if (parameter === UNMASKED_RENDERER_WEBGL) return 'Intel Iris OpenGL Engine';\n      return origGetParam.call(this, parameter);\n    };\n\n    // Hardware concurrency \u2014 pick from common values\n    const cpuCounts = [4, 6, 8, 8];\n    Object.defineProperty(navigator, 'hardwareConcurrency', {\n      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],\n      configurable: true,\n    });\n\n    // Device memory\n    Object.defineProperty(navigator, 'deviceMemory', {\n      get: () => [4, 8][Math.floor(Math.random() * 2)],\n      configurable: true,\n    });\n  }).catch(() => {});\n}\n\nexport async function infiniteScrollToBottom(\n  page: Page,\n  opts: { maxScrolls?: number; scrollDelay?: number; stabilizeMs?: number } = {},\n  rawPage?: any,\n): Promise<{ scrollCount: number; heightReached: number }> {\n  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;\n  let scrollCount = 0;\n  let stableScrollCount = 0;\n  let previousHeight = 0;\n\n  // One CDP session for wheel events (triggers IntersectionObserver-based virtual scrollers).\n  // React-window / react-virtualized / TanStack Virtual ignore window.scrollBy but respond to\n  // wheel events because they attach their own scroll listeners to the list container.\n  let cdpForWheel: any = null;\n  let viewW = 1280;\n  let viewH = 800;\n  if (rawPage && typeof rawPage.context === 'function') {\n    try {\n      const ctx = rawPage.context();\n      if (typeof ctx?.newCDPSession === 'function') {\n        cdpForWheel = await ctx.newCDPSession(rawPage);\n        const { result } = await cdpForWheel.send('Runtime.evaluate', {\n          expression: `({\"w\":window.innerWidth,\"h\":window.innerHeight})`,\n          returnByValue: true,\n        }).catch(() => ({ result: { value: null } }));\n        if (result?.value) { viewW = result.value.w ?? 1280; viewH = result.value.h ?? 800; }\n      }\n    } catch { cdpForWheel = null; }\n  }\n\n  try {\n    while (scrollCount < maxScrolls) {\n      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);\n\n      const scrollBy = 500 + Math.floor(Math.random() * 300);\n      // window.scrollBy handles standard scroll containers\n      await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});\n      // CDP wheel event triggers IntersectionObserver + virtual list scroll listeners\n      if (cdpForWheel) {\n        await cdpForWheel.send('Input.dispatchMouseEvent', {\n          type: 'mouseWheel',\n          x: Math.floor(viewW / 2),\n          y: Math.floor(viewH / 2),\n          deltaX: 0,\n          deltaY: scrollBy,\n          modifiers: 0,\n        }).catch(() => {});\n      }\n      await humanDelay(scrollDelay - 100, scrollDelay + 200);\n\n      // Use Node.js setTimeout \u2014 page.waitForTimeout may not be available on Stagehand proxy\n      await new Promise<void>(r => setTimeout(r, stabilizeMs));\n      scrollCount++;\n\n      const newHeight: number = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);\n      if (newHeight - previousHeight < 10) {\n        stableScrollCount++;\n        if (stableScrollCount >= 3) break; // no new content after 3 consecutive non-growing scrolls\n      } else {\n        stableScrollCount = 0;\n      }\n      previousHeight = newHeight;\n    }\n  } catch { /* non-fatal */ }\n\n  if (cdpForWheel) cdpForWheel.detach().catch(() => {});\n  return { scrollCount, heightReached: previousHeight };\n}\n\nexport async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {\n  const results: Array<{ src: string; content: string; markdown: string }> = [];\n  try {\n    const iframeSrcs: string[] = await page.evaluate(() =>\n      Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')\n    ).catch(() => []);\n\n    for (const frame of page.frames()) {\n      const src = frame.url();\n      if (!src || src === 'about:blank' || src === page.url()) continue;\n      try {\n        const content = await frame.content().catch(() => '');\n        if (!content || content.length < 50) continue;\n        // Simple text extraction from iframe HTML\n        const text = content.replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n          .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n          .replace(/<[^>]+>/g, ' ')\n          .replace(/\\s+/g, ' ')\n          .trim();\n        results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });\n      } catch { /* skip this frame */ }\n    }\n  } catch { /* non-fatal */ }\n  return results;\n}\n\nexport async function pierceShadowDom(page: Page): Promise<string> {\n  try {\n    return await page.evaluate(() => {\n      const parts: string[] = [];\n      function walkShadow(root: Element | ShadowRoot): void {\n        const shadow = (root as any).shadowRoot;\n        if (shadow) {\n          parts.push(shadow.innerHTML ?? '');\n          for (const child of [...shadow.querySelectorAll('*')]) walkShadow(child);\n        }\n        for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {\n          if ((child as any).shadowRoot) walkShadow(child);\n        }\n      }\n      walkShadow(document.body);\n      return parts\n        .join('\\n')\n        .replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n        .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n        .replace(/<[^>]+>/g, ' ')\n        .replace(/\\s+/g, ' ')\n        .trim()\n        .slice(0, 10_000);\n    }).catch(() => '');\n  } catch {\n    return '';\n  }\n}\n"],
+  "mappings": "AAGA,SAAS,SAAS,MAAc,OAAuB;AACrD,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,KAAK,KAAK,OAAO;AACvB,QAAM,IAAI,KAAK,KAAK,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;AAClE,SAAO,OAAO,IAAI;AACpB;AAEA,eAAsB,WAAW,MAAM,IAAI,MAAM,KAAoB;AACnE,QAAM,QAAQ,MAAM,OAAO;AAC3B,QAAM,SAAS,MAAM,OAAO;AAC5B,QAAM,QAAQ,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,SAAS,MAAM,KAAK,CAAC,CAAC;AAChE,QAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,KAAK,MAAM,KAAK,CAAC,CAAC;AACzD;AAEA,eAAsB,sBAAsB,MAAY,SAAiB,SAAgC;AACvG,MAAI;AACF,UAAM,QAAQ,KAAK;AACnB,UAAM,QAAQ,IAAI,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC;AAE9C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,SAAS,UAAU,MAAM,KAAK,OAAO,IAAI;AAC/C,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAC7D,UAAM,OAAO,SAAS,WAAW,KAAK,KAAK,OAAO,IAAI,OAAO;AAE7D,aAAS,IAAI,GAAG,KAAK,OAAO,KAAK;AAC/B,YAAM,IAAI,IAAI;AAEd,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,KAAK,IAAI,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI;AACvE,YAAM,MAAM,KAAK,KAAK,KAAK,OAAO,IAAI,OAAO,GAAG,KAAK,KAAK,OAAO,IAAI,OAAO,CAAC;AAC7E,UAAI,IAAI,MAAO,OAAM,WAAW,GAAG,EAAE;AAAA,IACvC;AAEA,UAAM,MAAM,KAAK,WAAW,KAAK,OAAO,IAAI,OAAO,GAAG,WAAW,KAAK,OAAO,IAAI,OAAO,CAAC;AACzF,UAAM,WAAW,IAAI,EAAE;AACvB,UAAM,MAAM,KAAK,SAAS,OAAO;AAAA,EACnC,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,mBAAmB,MAAY,UAAiC;AACpF,MAAI;AACF,UAAM,KAAK,MAAM,KAAK,EAAE,QAAQ;AAChC,QAAI,CAAC,GAAI;AACT,UAAM,MAAM,MAAM,GAAG,YAAY;AACjC,QAAI,CAAC,IAAK;AACV,UAAM,KAAK,IAAI,IAAI,IAAI,QAAQ;AAC/B,UAAM,KAAK,IAAI,IAAI,IAAI,SAAS;AAChC,UAAM,sBAAsB,MAAM,IAAI,EAAE;AACxC,UAAM,WAAW,KAAK,GAAG;AAAA,EAC3B,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,cAAc,MAAY,UAAiC;AAC/E,MAAI;AACF,UAAM,mBAAmB,MAAM,QAAQ;AACvC,UAAM,WAAW,IAAI,GAAG;AACxB,UAAM,KAAK,MAAM,UAAU,EAAE,OAAO,KAAK,MAAM,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;AAAA,EACpE,QAAQ;AAAA,EAAkB;AAC5B;AAEA,eAAsB,2BAA2B,MAA2B;AAC1E,QAAM,KAAK,cAAc,MAAM;AAE7B,UAAM,gBAAgB,kBAAkB,UAAU;AAClD,sBAAkB,UAAU,YAAY,YAAqC,MAAa;AACxF,YAAM,MAAM,KAAK,WAAW,IAAI;AAChC,UAAI,KAAK;AACP,cAAM,YAAY,IAAI,aAAa,GAAG,GAAG,KAAK,OAAO,KAAK,MAAM;AAChE,iBAAS,IAAI,GAAG,IAAI,UAAU,KAAK,QAAQ,KAAK,GAAG;AACjD,oBAAU,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,OAAO,IAAI,OAAQ,KAAK,OAAO,IAAI,MAAM,IAAI,KAAM,EAAE,CAAC;AAAA,QAChI;AACA,YAAI,aAAa,WAAW,GAAG,CAAC;AAAA,MAClC;AACA,aAAO,cAAc,MAAM,MAAM,IAAyC;AAAA,IAC5E;AAGA,UAAM,eAAe,sBAAsB,UAAU;AACrD,0BAAsB,UAAU,eAAe,SAAsC,WAAmB;AACtG,YAAM,wBAAwB;AAC9B,YAAM,0BAA0B;AAChC,UAAI,cAAc,sBAAuB,QAAO;AAChD,UAAI,cAAc,wBAAyB,QAAO;AAClD,aAAO,aAAa,KAAK,MAAM,SAAS;AAAA,IAC1C;AAGA,UAAM,YAAY,CAAC,GAAG,GAAG,GAAG,CAAC;AAC7B,WAAO,eAAe,WAAW,uBAAuB;AAAA,MACtD,KAAK,MAAM,UAAU,KAAK,MAAM,KAAK,OAAO,IAAI,UAAU,MAAM,CAAC;AAAA,MACjE,cAAc;AAAA,IAChB,CAAC;AAGD,WAAO,eAAe,WAAW,gBAAgB;AAAA,MAC/C,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC;AAAA,MAC/C,cAAc;AAAA,IAChB,CAAC;AAAA,EACH,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACnB;AAEA,eAAsB,uBACpB,MACA,OAA4E,CAAC,GAC7E,SACyD;AACzD,QAAM,EAAE,aAAa,IAAI,cAAc,KAAK,cAAc,IAAI,IAAI;AAClE,MAAI,cAAc;AAClB,MAAI,oBAAoB;AACxB,MAAI,iBAAiB;AAKrB,MAAI,cAAmB;AACvB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,MAAI,WAAW,OAAO,QAAQ,YAAY,YAAY;AACpD,QAAI;AACF,YAAM,MAAM,QAAQ,QAAQ;AAC5B,UAAI,OAAO,KAAK,kBAAkB,YAAY;AAC5C,sBAAc,MAAM,IAAI,cAAc,OAAO;AAC7C,cAAM,EAAE,OAAO,IAAI,MAAM,YAAY,KAAK,oBAAoB;AAAA,UAC5D,YAAY;AAAA,UACZ,eAAe;AAAA,QACjB,CAAC,EAAE,MAAM,OAAO,EAAE,QAAQ,EAAE,OAAO,KAAK,EAAE,EAAE;AAC5C,YAAI,QAAQ,OAAO;AAAE,kBAAQ,OAAO,MAAM,KAAK;AAAM,kBAAQ,OAAO,MAAM,KAAK;AAAA,QAAK;AAAA,MACtF;AAAA,IACF,QAAQ;AAAE,oBAAc;AAAA,IAAM;AAAA,EAChC;AAEA,MAAI;AACF,WAAO,cAAc,YAAY;AAC/B,uBAAiB,MAAM,KAAK,SAAS,MAAM,SAAS,gBAAgB,YAAY,EAAE,MAAM,MAAM,CAAC;AAE/F,YAAM,WAAW,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAErD,YAAM,KAAK,SAAS,CAAC,OAAe,OAAO,SAAS,GAAG,EAAE,GAAG,QAAQ,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAEpF,UAAI,aAAa;AACf,cAAM,YAAY,KAAK,4BAA4B;AAAA,UACjD,MAAM;AAAA,UACN,GAAG,KAAK,MAAM,QAAQ,CAAC;AAAA,UACvB,GAAG,KAAK,MAAM,QAAQ,CAAC;AAAA,UACvB,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,WAAW;AAAA,QACb,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM,WAAW,cAAc,KAAK,cAAc,GAAG;AAGrD,YAAM,IAAI,QAAc,OAAK,WAAW,GAAG,WAAW,CAAC;AACvD;AAEA,YAAM,YAAoB,MAAM,KAAK,SAAS,MAAM,SAAS,gBAAgB,YAAY,EAAE,MAAM,MAAM,CAAC;AACxG,UAAI,YAAY,iBAAiB,IAAI;AACnC;AACA,YAAI,qBAAqB,EAAG;AAAA,MAC9B,OAAO;AACL,4BAAoB;AAAA,MACtB;AACA,uBAAiB;AAAA,IACnB;AAAA,EACF,QAAQ;AAAA,EAAkB;AAE1B,MAAI,YAAa,aAAY,OAAO,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACpD,SAAO,EAAE,aAAa,eAAe,eAAe;AACtD;AAEA,eAAsB,qBAAqB,MAAgF;AACzH,QAAM,UAAqE,CAAC;AAC5E,MAAI;AACF,UAAM,aAAuB,MAAM,KAAK;AAAA,MAAS,MAC/C,MAAM,KAAK,SAAS,iBAAiB,QAAQ,CAAC,EAAE,IAAI,OAAK,EAAE,aAAa,KAAK,KAAK,EAAE,aAAa,UAAU,KAAK,EAAE;AAAA,IACpH,EAAE,MAAM,MAAM,CAAC,CAAC;AAEhB,eAAW,SAAS,KAAK,OAAO,GAAG;AACjC,YAAM,MAAM,MAAM,IAAI;AACtB,UAAI,CAAC,OAAO,QAAQ,iBAAiB,QAAQ,KAAK,IAAI,EAAG;AACzD,UAAI;AACF,cAAM,UAAU,MAAM,MAAM,QAAQ,EAAE,MAAM,MAAM,EAAE;AACpD,YAAI,CAAC,WAAW,QAAQ,SAAS,GAAI;AAErC,cAAM,OAAO,QAAQ,QAAQ,qCAAqC,EAAE,EACjE,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACR,gBAAQ,KAAK,EAAE,KAAK,SAAS,QAAQ,MAAM,GAAG,GAAM,GAAG,UAAU,KAAK,MAAM,GAAG,GAAK,EAAE,CAAC;AAAA,MACzF,QAAQ;AAAA,MAAwB;AAAA,IAClC;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAEA,eAAsB,gBAAgB,MAA6B;AACjE,MAAI;AACF,WAAO,MAAM,KAAK,SAAS,MAAM;AAC/B,YAAM,QAAkB,CAAC;AACzB,eAAS,WAAW,MAAkC;AACpD,cAAM,SAAU,KAAa;AAC7B,YAAI,QAAQ;AACV,gBAAM,KAAK,OAAO,aAAa,EAAE;AACjC,qBAAW,SAAS,CAAC,GAAG,OAAO,iBAAiB,GAAG,CAAC,EAAG,YAAW,KAAK;AAAA,QACzE;AACA,mBAAW,SAAS,MAAM,KAAK,KAAK,iBAAiB,yBAAyB,CAAC,GAAG;AAChF,cAAK,MAAc,WAAY,YAAW,KAAK;AAAA,QACjD;AAAA,MACF;AACA,iBAAW,SAAS,IAAI;AACxB,aAAO,MACJ,KAAK,IAAI,EACT,QAAQ,qCAAqC,EAAE,EAC/C,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,YAAY,GAAG,EACvB,QAAQ,QAAQ,GAAG,EACnB,KAAK,EACL,MAAM,GAAG,GAAM;AAAA,IACpB,CAAC,EAAE,MAAM,MAAM,EAAE;AAAA,EACnB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
   "names": []
 }

package/dist/url-pattern-completer.js

@@ -1,7 +1,24 @@
 import { logger as rootLogger } from "./logger.js";
 const logger = rootLogger.child({ module: "url-pattern-completer" });
+const SUPPORTED = /* @__PURE__ */ new Set(["openai", "anthropic"]);
+function resolveTextLLM(llmCfg) {
+  if (llmCfg?.provider === "openai" && llmCfg.apiKey) {
+    return { provider: "openai", apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };
+  }
+  if (llmCfg?.provider === "anthropic" && llmCfg.apiKey) {
+    return { provider: "anthropic", apiKey: llmCfg.apiKey, model: llmCfg.model };
+  }
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: "openai", apiKey: process.env.OPENAI_API_KEY, model: "gpt-4o-mini" };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { provider: "anthropic", apiKey: process.env.ANTHROPIC_API_KEY, model: "claude-haiku-4-5-20251001" };
+  }
+  return null;
+}
 async function runUrlPatternCompletion(discoveredUrls, appUrl, llmCfg) {
-  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];
+  const effective = resolveTextLLM(llmCfg);
+  if (!effective || discoveredUrls.length < 2) return [];
   const origin = (() => {
     try {
       return new URL(appUrl).origin;
@@ -19,7 +36,8 @@ async function runUrlPatternCompletion(discoveredUrls, appUrl, llmCfg) {
     }
   }).slice(0, 60);
   if (internalUrls.length < 2) return [];
-  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
+  const systemPrompt = "You are a URL pattern analyst. Always respond with valid JSON only.";
+  const userPrompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
 
 Discovered URLs:
 ${internalUrls.map((u) => {
@@ -42,17 +60,33 @@ Rules:
 Respond with ONLY valid JSON, no explanation:
 {"candidates":[{"path":"/settings/billing","confidence":0.85,"reason":"settings subpage pattern from /settings/profile"},{"path":"/admin/users","confidence":0.6,"reason":"admin prefix implied by /admin/dashboard"}]}`;
   try {
-    const Anthropic = (await import("@anthropic-ai/sdk")).default;
-    const client = new Anthropic({ apiKey: llmCfg.apiKey });
-    const response = await client.messages.create({
-      model: llmCfg.model ?? "claude-haiku-4-5-20251001",
-      max_tokens: 1024,
-      temperature: 0,
-      // @ts-ignore: cache_control supported in beta
-      system: [{ type: "text", text: "You are a URL pattern analyst. Always respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
-      messages: [{ role: "user", content: prompt }]
-    });
-    const text = response.content[0]?.text ?? "";
+    let text = "";
+    if (effective.provider === "openai") {
+      const { OpenAI } = await import("openai");
+      const client = new OpenAI({ apiKey: effective.apiKey, ...effective.baseUrl ? { baseURL: effective.baseUrl } : {} });
+      const response = await client.chat.completions.create({
+        model: effective.model ?? "gpt-4o-mini",
+        max_tokens: 1024,
+        temperature: 0,
+        messages: [
+          { role: "system", content: systemPrompt },
+          { role: "user", content: userPrompt }
+        ]
+      });
+      text = response.choices[0]?.message?.content ?? "";
+    } else {
+      const Anthropic = (await import("@anthropic-ai/sdk")).default;
+      const client = new Anthropic({ apiKey: effective.apiKey });
+      const response = await client.messages.create({
+        model: effective.model ?? "claude-haiku-4-5-20251001",
+        max_tokens: 1024,
+        temperature: 0,
+        // @ts-ignore: cache_control supported in beta
+        system: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }],
+        messages: [{ role: "user", content: userPrompt }]
+      });
+      text = response.content[0]?.text ?? "";
+    }
     const jsonMatch = text.match(/\{[\s\S]*\}/);
     if (!jsonMatch) return [];
     const parsed = JSON.parse(jsonMatch[0]);
@@ -75,7 +109,7 @@ Respond with ONLY valid JSON, no explanation:
       }
     }
     const hits = results.filter((r) => r.status === "hit");
-    logger.info({ total: results.length, hits: hits.length }, "[url-completer] pattern completion done");
+    logger.info({ total: results.length, hits: hits.length, provider: effective.provider }, "[url-completer] pattern completion done");
     return results;
   } catch (err) {
     logger.warn({ err: String(err) }, "[url-completer] LLM call failed \u2014 skipping pattern completion");

package/dist/url-pattern-completer.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/url-pattern-completer.ts"],
-  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      temperature: 0,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
-  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA,MACZ,aAAa;AAAA;AAAA,MAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
+  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n// Only these two providers are supported by this module (they use standard SDK calls).\n// NVIDIA, groq, ollama, openrouter, etc. are skipped \u2014 fall through to env fallback.\nconst SUPPORTED = new Set(['openai', 'anthropic']);\n\n/**\n * Resolve the effective LLM to use. Priority: openai \u2192 anthropic \u2192 skip others.\n * Falls back to env vars when the platform-configured provider is unsupported.\n */\nfunction resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null {\n  // Use platform config if it's a supported provider with a key\n  if (llmCfg?.provider === 'openai' && llmCfg.apiKey) {\n    return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };\n  }\n  if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) {\n    return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model };\n  }\n  // Platform provider is unsupported (nvidia, groq, etc.) or no key \u2014 fall back to env vars.\n  // OpenAI takes priority over Anthropic.\n  if (process.env.OPENAI_API_KEY) {\n    return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' };\n  }\n  if (process.env.ANTHROPIC_API_KEY) {\n    return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' };\n  }\n  return null;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  const effective = resolveTextLLM(llmCfg);\n  if (!effective || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const systemPrompt = 'You are a URL pattern analyst. Always respond with valid JSON only.';\n  const userPrompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    let text = '';\n\n    if (effective.provider === 'openai') {\n      const { OpenAI } = await import('openai');\n      const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) });\n      const response = await client.chat.completions.create({\n        model: effective.model ?? 'gpt-4o-mini',\n        max_tokens: 1024,\n        temperature: 0,\n        messages: [\n          { role: 'system', content: systemPrompt },\n          { role: 'user', content: userPrompt },\n        ],\n      });\n      text = response.choices[0]?.message?.content ?? '';\n    } else {\n      const Anthropic = (await import('@anthropic-ai/sdk')).default;\n      const client = new Anthropic({ apiKey: effective.apiKey });\n      const response = await client.messages.create({\n        model: effective.model ?? 'claude-haiku-4-5-20251001',\n        max_tokens: 1024,\n        temperature: 0,\n        // @ts-ignore: cache_control supported in beta\n        system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],\n        messages: [{ role: 'user', content: userPrompt }],\n      });\n      text = (response.content[0] as any)?.text ?? '';\n    }\n\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length, provider: effective.provider }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
+  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAkBnE,MAAM,YAAY,oBAAI,IAAI,CAAC,UAAU,WAAW,CAAC;AAMjD,SAAS,eAAe,QAAiI;AAEvJ,MAAI,QAAQ,aAAa,YAAY,OAAO,QAAQ;AAClD,WAAO,EAAE,UAAU,UAAU,QAAQ,OAAO,QAAQ,OAAO,OAAO,OAAO,SAAS,OAAO,QAAQ;AAAA,EACnG;AACA,MAAI,QAAQ,aAAa,eAAe,OAAO,QAAQ;AACrD,WAAO,EAAE,UAAU,aAAa,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM;AAAA,EAC7E;AAGA,MAAI,QAAQ,IAAI,gBAAgB;AAC9B,WAAO,EAAE,UAAU,UAAU,QAAQ,QAAQ,IAAI,gBAAgB,OAAO,cAAc;AAAA,EACxF;AACA,MAAI,QAAQ,IAAI,mBAAmB;AACjC,WAAO,EAAE,UAAU,aAAa,QAAQ,QAAQ,IAAI,mBAAmB,OAAO,4BAA4B;AAAA,EAC5G;AACA,SAAO;AACT;AAMA,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,QAAM,YAAY,eAAe,MAAM;AACvC,MAAI,CAAC,aAAa,eAAe,SAAS,EAAG,QAAO,CAAC;AAErD,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,eAAe;AACrB,QAAM,aAAa;AAAA;AAAA;AAAA,EAGnB,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,QAAI,OAAO;AAEX,QAAI,UAAU,aAAa,UAAU;AACnC,YAAM,EAAE,OAAO,IAAI,MAAM,OAAO,QAAQ;AACxC,YAAM,SAAS,IAAI,OAAO,EAAE,QAAQ,UAAU,QAAQ,GAAI,UAAU,UAAU,EAAE,SAAS,UAAU,QAAQ,IAAI,CAAC,EAAG,CAAC;AACpH,YAAM,WAAW,MAAM,OAAO,KAAK,YAAY,OAAO;AAAA,QACpD,OAAO,UAAU,SAAS;AAAA,QAC1B,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,UAAU;AAAA,UACR,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,UACxC,EAAE,MAAM,QAAQ,SAAS,WAAW;AAAA,QACtC;AAAA,MACF,CAAC;AACD,aAAO,SAAS,QAAQ,CAAC,GAAG,SAAS,WAAW;AAAA,IAClD,OAAO;AACL,YAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,YAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,UAAU,OAAO,CAAC;AACzD,YAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,QAC5C,OAAO,UAAU,SAAS;AAAA,QAC1B,YAAY;AAAA,QACZ,aAAa;AAAA;AAAA,QAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,cAAc,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,QACnF,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,WAAW,CAAC;AAAA,MAClD,CAAC;AACD,aAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AAAA,IAC/C;AAEA,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,QAAQ,UAAU,UAAU,SAAS,GAAG,yCAAyC;AACjI,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
   "names": []
 }

package/dist/visual-diff-verdict.js

@@ -1,6 +1,8 @@
 function pHashVerdict(pHashDistance, changeLevel, changes) {
   const changeType = changeLevel === "IDENTICAL" ? "NONE" : changeLevel === "MINOR" ? "MINOR" : changeLevel === "MODERATE" ? "SIGNIFICANT" : "CRITICAL";
-  const recommendation = pHashDistance === 0 ? "APPROVE" : pHashDistance < 5 ? "APPROVE" : pHashDistance < 20 ? "REVIEW" : "FLAG";
+  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? "") || 5;
+  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? "") || 20;
+  const recommendation = pHashDistance === 0 ? "APPROVE" : pHashDistance < approveThreshold ? "APPROVE" : pHashDistance < reviewThreshold ? "REVIEW" : "FLAG";
   return { changeType, recommendation, changes };
 }
 function mergeAiVisualVerdict(pHashResult, aiResult, screenName) {

package/dist/visual-diff-verdict.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/visual-diff-verdict.ts"],
-  "sourcesContent": ["/**\n * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested\n * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and\n * transitively @detiq/database, which instantiates a PrismaClient at module load and\n * so requires a live DATABASE_URL just to import the module).\n */\n\nexport interface VisualDiffVerdict {\n  changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';\n  recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';\n  changes: Array<{ location: string; severity: string; description: string }>;\n  bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };\n  aiSummary?: string;\n}\n\n/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */\nexport function pHashVerdict(\n  pHashDistance: number,\n  changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',\n  changes: Array<{ location: string; severity: string; description: string }>,\n): VisualDiffVerdict {\n  const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'\n    : changeLevel === 'MINOR' ? 'MINOR'\n      : changeLevel === 'MODERATE' ? 'SIGNIFICANT'\n        : 'CRITICAL';\n  const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'\n    : pHashDistance < 5 ? 'APPROVE'\n      : pHashDistance < 20 ? 'REVIEW'\n        : 'FLAG';\n  return { changeType, recommendation, changes };\n}\n\n/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,\n *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review\n *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards\n *  as NONE/MINOR \u2014 only what survives AI review is worth a draft). */\nexport function mergeAiVisualVerdict(\n  pHashResult: VisualDiffVerdict,\n  aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },\n  screenName: string,\n): VisualDiffVerdict {\n  if (!aiResult.changeType) return pHashResult;\n  const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];\n  const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];\n  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;\n  const aiSummary = aiResult.summary ?? undefined;\n  const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')\n    ? {\n        title: `Visual regression on \"${screenName}\"`,\n        severity: changeType === 'CRITICAL' ? 'High' : 'Medium',\n        steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],\n        expected: 'Screen appearance should match the stored baseline.',\n        actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,\n      }\n    : undefined;\n  return { changeType, recommendation, changes, bugReportDraft, aiSummary };\n}\n"],
-  "mappings": "AAgBO,SAAS,aACd,eACA,aACA,SACmB;AACnB,QAAM,aAA8C,gBAAgB,cAAc,SAC9E,gBAAgB,UAAU,UACxB,gBAAgB,aAAa,gBAC3B;AACR,QAAM,iBAAsD,kBAAkB,IAAI,YAC9E,gBAAgB,IAAI,YAClB,gBAAgB,KAAK,WACnB;AACR,SAAO,EAAE,YAAY,gBAAgB,QAAQ;AAC/C;AAMO,SAAS,qBACd,aACA,UACA,YACmB;AACnB,MAAI,CAAC,SAAS,WAAY,QAAO;AACjC,QAAM,aAAa,SAAS;AAC5B,QAAM,iBAAiB,SAAS;AAChC,QAAM,UAAU,MAAM,QAAQ,SAAS,OAAO,KAAK,SAAS,QAAQ,SAAS,IAAI,SAAS,UAAU,YAAY;AAChH,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,iBAAkB,eAAe,cAAc,eAAe,gBAChE;AAAA,IACE,OAAO,yBAAyB,UAAU;AAAA,IAC1C,UAAU,eAAe,aAAa,SAAS;AAAA,IAC/C,OAAO,CAAC,0BAA0B,6CAA6C;AAAA,IAC/E,UAAU;AAAA,IACV,QAAQ,aAAa,mCAAmC,SAAS,UAAU,QAAQ,CAAC,CAAC;AAAA,EACvF,IACA;AACJ,SAAO,EAAE,YAAY,gBAAgB,SAAS,gBAAgB,UAAU;AAC1E;",
+  "sourcesContent": ["/**\n * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested\n * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and\n * transitively @detiq/database, which instantiates a PrismaClient at module load and\n * so requires a live DATABASE_URL just to import the module).\n */\n\nexport interface VisualDiffVerdict {\n  changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';\n  recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';\n  changes: Array<{ location: string; severity: string; description: string }>;\n  bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };\n  aiSummary?: string;\n}\n\n/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */\nexport function pHashVerdict(\n  pHashDistance: number,\n  changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',\n  changes: Array<{ location: string; severity: string; description: string }>,\n): VisualDiffVerdict {\n  const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'\n    : changeLevel === 'MINOR' ? 'MINOR'\n      : changeLevel === 'MODERATE' ? 'SIGNIFICANT'\n        : 'CRITICAL';\n  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? '') || 5;\n  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? '') || 20;\n  const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'\n    : pHashDistance < approveThreshold ? 'APPROVE'\n      : pHashDistance < reviewThreshold ? 'REVIEW'\n        : 'FLAG';\n  return { changeType, recommendation, changes };\n}\n\n/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,\n *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review\n *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards\n *  as NONE/MINOR \u2014 only what survives AI review is worth a draft). */\nexport function mergeAiVisualVerdict(\n  pHashResult: VisualDiffVerdict,\n  aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },\n  screenName: string,\n): VisualDiffVerdict {\n  if (!aiResult.changeType) return pHashResult;\n  const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];\n  const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];\n  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;\n  const aiSummary = aiResult.summary ?? undefined;\n  const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')\n    ? {\n        title: `Visual regression on \"${screenName}\"`,\n        severity: changeType === 'CRITICAL' ? 'High' : 'Medium',\n        steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],\n        expected: 'Screen appearance should match the stored baseline.',\n        actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,\n      }\n    : undefined;\n  return { changeType, recommendation, changes, bugReportDraft, aiSummary };\n}\n"],
+  "mappings": "AAgBO,SAAS,aACd,eACA,aACA,SACmB;AACnB,QAAM,aAA8C,gBAAgB,cAAc,SAC9E,gBAAgB,UAAU,UACxB,gBAAgB,aAAa,gBAC3B;AACR,QAAM,mBAAmB,SAAS,QAAQ,IAAI,yCAAyC,EAAE,KAAK;AAC9F,QAAM,kBAAkB,SAAS,QAAQ,IAAI,wCAAwC,EAAE,KAAK;AAC5F,QAAM,iBAAsD,kBAAkB,IAAI,YAC9E,gBAAgB,mBAAmB,YACjC,gBAAgB,kBAAkB,WAChC;AACR,SAAO,EAAE,YAAY,gBAAgB,QAAQ;AAC/C;AAMO,SAAS,qBACd,aACA,UACA,YACmB;AACnB,MAAI,CAAC,SAAS,WAAY,QAAO;AACjC,QAAM,aAAa,SAAS;AAC5B,QAAM,iBAAiB,SAAS;AAChC,QAAM,UAAU,MAAM,QAAQ,SAAS,OAAO,KAAK,SAAS,QAAQ,SAAS,IAAI,SAAS,UAAU,YAAY;AAChH,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,iBAAkB,eAAe,cAAc,eAAe,gBAChE;AAAA,IACE,OAAO,yBAAyB,UAAU;AAAA,IAC1C,UAAU,eAAe,aAAa,SAAS;AAAA,IAC/C,OAAO,CAAC,0BAA0B,6CAA6C;AAAA,IAC/E,UAAU;AAAA,IACV,QAAQ,aAAa,mCAAmC,SAAS,UAAU,QAAQ,CAAC,CAAC;AAAA,EACvF,IACA;AACJ,SAAO,EAAE,YAAY,gBAAgB,SAAS,gBAAgB,UAAU;AAC1E;",
   "names": []
 }