@detiq/crawler 1.1.5 → 1.1.7

package/dist/sitemap.js

@@ -60,9 +60,11 @@ async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
   if (!parsed) return [];
   const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
   const collected = [...pageUrls];
-  for (const subUrl of subSitemapUrls) {
-    if (collected.length >= MAX_URLS) break;
-    collected.push(...await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher));
+  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+    );
+    for (const sub of subResults) collected.push(...sub);
   }
   return [...new Set(collected)].slice(0, MAX_URLS);
 }

package/dist/sitemap.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/sitemap.ts"],
-  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  for (const subUrl of subSitemapUrls) {\n    if (collected.length >= MAX_URLS) break;\n    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
-  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,aAAW,UAAU,gBAAgB;AACnC,QAAI,UAAU,UAAU,SAAU;AAClC,cAAU,KAAK,GAAI,MAAM,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAE;AAAA,EACnF;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
+  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {\n    const subResults = await Promise.all(\n      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))\n    );\n    for (const sub of subResults) collected.push(...sub);\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
+  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,MAAI,eAAe,SAAS,KAAK,UAAU,SAAS,UAAU;AAC5D,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,eAAe,MAAM,GAAG,EAAE,EAAE,IAAI,YAAU,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAC;AAAA,IACnG;AACA,eAAW,OAAO,WAAY,WAAU,KAAK,GAAG,GAAG;AAAA,EACrD;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
   "names": []
 }

package/dist/url-pattern-completer.js

@@ -47,6 +47,7 @@ Respond with ONLY valid JSON, no explanation:
     const response = await client.messages.create({
       model: llmCfg.model ?? "claude-haiku-4-5-20251001",
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore: cache_control supported in beta
       system: [{ type: "text", text: "You are a URL pattern analyst. Always respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
       messages: [{ role: "user", content: prompt }]

package/dist/url-pattern-completer.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/url-pattern-completer.ts"],
-  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
-  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
+  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      temperature: 0,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
+  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA,MACZ,aAAa;AAAA;AAAA,MAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.5",
+  "version": "1.1.7",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -28,7 +28,7 @@ import { z } from 'zod';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { AppBrain, recordLLMCall } from '@detiq/app-brain';
-import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
+import { aiVisualCompare, askTracked, deterministicSeed, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
 import { config } from '@detiq/core';
 import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
 import type { Credentials } from './auth.js';
@@ -1125,6 +1125,14 @@ export async function crawlProject(opts: {
     await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
     await waitForStableDOM(page);
 
+    // Capture homepage fingerprint for soft-404 detection. SPA catch-all routes return HTTP 200
+    // but render identical homepage content for every unrecognised path. We fingerprint here
+    // (pre-auth, stable DOM) and skip any BFS page that matches during the main crawl.
+    const homeFingerprintTitle = await page.title().catch(() => '');
+    const homeFingerprintElemCount = await page.evaluate(() =>
+      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+
     // Dedicated logged-out/auth discovery phase. This runs before credentials or
     // storageState are applied, so public pages and auth recovery flows remain
     // visible even when the main crawl later becomes authenticated.
@@ -1669,6 +1677,25 @@ export async function crawlProject(opts: {
             await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
           }
 
+          // Wait for SPA JS to render components before extracting elements or running soft-404 check.
+          // domcontentloaded fires before JS executes — networkidle ensures React/Vue/Angular hydration.
+          await workerPage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+
+          // Soft-404 detection for lite-worker: skip SPA catch-all URLs that mirror homepage
+          if (homeFingerprintElemCount >= 0 && url !== startUrl) {
+            const wTitle = await workerPage.title().catch(() => '');
+            if (wTitle === homeFingerprintTitle && wTitle !== '') {
+              const wElemCount = await workerPage.evaluate(() =>
+                document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+              ).catch(() => -1);
+              if (wElemCount >= 0 && wElemCount === homeFingerprintElemCount) {
+                console.log(`[lite-worker] skip ${url} — soft-404 (homepage fingerprint match)`);
+                visited.set(canonUrl, '_soft404_');
+                continue;
+              }
+            }
+          }
+
           const pathParts = new URL(url).pathname.split('/').filter(Boolean);
           const name = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
 
@@ -2157,6 +2184,18 @@ export async function crawlProject(opts: {
           seenDomHashes.add(preDomHash); // register so future 404 pages also hit hash-dedup
           continue;
         }
+        // Soft-404: SPA catch-all routes return 200 but render identical homepage content.
+        // Skip any non-homepage URL whose title + interactive element count matches the homepage.
+        if (homeFingerprintElemCount >= 0 && url !== startUrl && pageTitle === homeFingerprintTitle && pageTitle !== '') {
+          const pElemCount = await page.evaluate(() =>
+            document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+          ).catch(() => -1);
+          if (pElemCount >= 0 && pElemCount === homeFingerprintElemCount) {
+            console.log(`[crawler] skip ${url} — soft-404 (homepage fingerprint match, title="${pageTitle}" elems=${pElemCount})`);
+            seenDomHashes.add(preDomHash);
+            continue;
+          }
+        }
         // Do NOT add real pages to seenDomHashes — server-rendered sites (WooCommerce, WordPress)
         // use the same template for many legitimate pages; registering the hash would skip them all.
 
@@ -2537,11 +2576,14 @@ export async function crawlProject(opts: {
                   const snippet = extracted.markdown.slice(0, 1200);
                   // Skip LLM when content is too thin — output would be useless
                   if (snippet.trim().length < 80) return undefined;
+                  const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ''}:page-summary:${url}`);
                   const { text, usage } = await askTracked(
                     'You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.',
                     `Screen: "${name}"\nURL: ${url}\n\nContent:\n${snippet}`,
                     llmCfg,
                     200,
+                    undefined,
+                    { temperature: 0, seed: summarySeed },
                   );
                   if (usage) {
                     recordLLMCall({
@@ -4045,13 +4087,23 @@ async function domFallbackElements(page: any): Promise<Array<{ meaning: string;
  *  Headings, meta tags, cards, chips, FAQ items, nav/footer structure, section text.
  *  Previously fallback-only; now always runs alongside domFallbackElements so
  *  the test generator sees ALL page content, not just interactive widgets. */
-async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string }>> {
+async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number> }>> {
   return page.evaluate(() => {
-    const out: { meaning: string; role: string; notes?: string }[] = [];
+    const out: { meaning: string; role: string; notes?: string; boundingRect?: Record<string, number> }[] = [];
     const seen = new Set<string>();
-    const addUniq = (item: { meaning: string; role: string; notes?: string }) => {
+    const captureRect = (el: Element): Record<string, number> | undefined => {
+      try {
+        const r = el.getBoundingClientRect();
+        if (r.width === 0 && r.height === 0) return undefined;
+        return { x: r.x, y: r.y, width: r.width, height: r.height, top: r.top, right: r.right, bottom: r.bottom, left: r.left };
+      } catch { return undefined; }
+    };
+    const addUniq = (item: { meaning: string; role: string; notes?: string }, el?: Element) => {
       const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
-      if (!seen.has(key) && item.meaning.trim().length > 1) { seen.add(key); out.push(item); }
+      if (!seen.has(key) && item.meaning.trim().length > 1) {
+        seen.add(key);
+        out.push(el ? { ...item, boundingRect: captureRect(el) } : item);
+      }
     };
     const isVisible = (el: Element) => {
       const s = window.getComputedStyle(el);
@@ -4072,7 +4124,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h: any) => {
       if (!isVisible(h)) return;
       const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
-      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
+      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) }, h);
     });
 
     // 3. FAQ / accordion — <details>/<summary>, ARIA pattern, and class-name scan
@@ -4113,7 +4165,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('[class*="card" i]:not(nav):not(header):not(footer), [class*="chip" i], [class*="badge" i], [class*="tag" i], [class*="pill" i]').forEach((el: any) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' });
+      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' }, el);
     });
 
     // 5. Feature / service list items inside main content (capped per category)
@@ -4123,14 +4175,14 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
       if (listItemCount >= 30) return;
       if (!isVisible(li)) return;
       const text = li.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }); listItemCount++; }
+      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }, li); listItemCount++; }
     });
 
     // 6. First paragraph in each section (key descriptive copy)
     mainContent.querySelectorAll('section > p:first-of-type, article > p:first-of-type, [class*="section" i] > p:first-of-type').forEach((p: any) => {
       if (!isVisible(p)) return;
       const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 250);
-      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' });
+      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' }, p);
     });
 
     // 7. CTA section text (standalone paragraphs near buttons that aren't form fields)
@@ -4562,7 +4614,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     document.querySelectorAll('[role="alert"], [role="status"], [role="banner"]:not(header), .alert, .notice, .notification, .warning-banner, .error-banner, .success-banner, [class*="alert-" i]:not(button), [class*="toast" i]:not(button), [class*="snackbar" i]').forEach((el: any) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
-      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) });
+      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) }, el);
     });
 
     // 27. Pricing / plan cards — structured pricing content
@@ -5339,6 +5391,21 @@ async function runAuthDiscoveryPhase(
     try { enqueue(new URL(p, origin).toString(), 'seed_path'); } catch { /* ignore bad seed */ }
   }
 
+  // Capture homepage DOM fingerprint for soft-404 detection.
+  // SPA catch-all sites return 200 for every path but render identical content.
+  // If a seeded path produces the same element count + title as the homepage, it's
+  // a phantom route — skip it so auth paths don't pollute the discovered URL list.
+  let homeTitle = '';
+  let homeElemCount = -1;
+  try {
+    await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
+    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
+    homeTitle = await page.title().catch(() => '');
+    homeElemCount = await page.evaluate(() =>
+      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+  } catch { /* best-effort */ }
+
   let captured = 0;
   let ordinal = 0;
   await onProgress?.(10, 'Discovering public and auth flows');
@@ -5352,6 +5419,16 @@ async function runAuthDiscoveryPhase(
       await waitForStableDOM(page);
       await acceptCookieConsent(page, { waitMs: 400 }).catch(() => { });
 
+      // Soft-404 detection: SPA returns 200 for unknown paths but renders homepage.
+      // Skip if title + element count match the homepage fingerprint exactly.
+      if (current.source === 'seed_path' && homeElemCount >= 0) {
+        const pageTitle = await page.title().catch(() => '');
+        const elemCount = await page.evaluate(() =>
+          document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+        ).catch(() => -1);
+        if (pageTitle === homeTitle && elemCount === homeElemCount) continue;
+      }
+
       const finalUrl = page.url?.() ?? current.url;
       const captureKey = normalizeAuthUrl(finalUrl);
       if (!capturedKeys.has(captureKey)) {

package/src/failure-analysis-agent.ts

@@ -1,5 +1,5 @@
 import { readFileSync } from 'node:fs';
-import { askTrackedWithImages, askTracked } from '@detiq/agents';
+import { askTrackedWithImages, askTracked, deterministicSeed } from '@detiq/agents';
 import type { LLMConfig } from '@detiq/agents';
 import { AppBrain } from '@detiq/app-brain';
 
@@ -41,7 +41,8 @@ Error: ${ctx.error}${consoleSection}${networkSection}`;
       } catch { /* fall through to text-only */ }
     }
 
-    const result = await askTracked('', userText, llmCfg, 256);
+    const failureSeed = deterministicSeed(`${ctx.tenantId ?? ''}:failure:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+    const result = await askTracked('', userText, llmCfg, 256, undefined, { temperature: 0, seed: failureSeed });
     return result.text || null;
   } catch {
     return null;

package/src/gap-pass.ts

@@ -75,6 +75,7 @@ Respond with ONLY valid JSON, no markdown:
     const response = await client.messages.create({
       model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore
       system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: prompt }],

package/src/job-queue.ts

@@ -1273,7 +1273,7 @@ async function processGenerateTestsFromReqsJob(jobId: string, config: any) {
   try {
     await updateJobProgress(tenantId, jobId, 5, 'Loading requirements');
 
-    const { askTracked, getTaskAgentLLMConfig } = await import('@detiq/agents');
+    const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import('@detiq/agents');
     const llmCfg = await getTaskAgentLLMConfig(tenantId, 'testGeneration');
 
     const all = await AppBrain.listRequirements(tenantId, projectId, {}) as any[];
@@ -1321,7 +1321,8 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
     let parsed: { cases: any[] } = { cases: [] };
     try {
-      const { text, usage } = await askTracked(system, user, llmCfg);
+      const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
+      const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
       if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => {});
       const json = text.match(/\{[\s\S]*\}/)?.[0];
       if (json) parsed = JSON.parse(json);

package/src/url-pattern-completer.ts

@@ -72,6 +72,7 @@ Respond with ONLY valid JSON, no explanation:
     const response = await client.messages.create({
       model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore: cache_control supported in beta
       system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
       messages: [{ role: 'user', content: prompt }],