@detiq/crawler 1.1.6 → 1.1.7

package/dist/sitemap.js

@@ -60,9 +60,11 @@ async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
   if (!parsed) return [];
   const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
   const collected = [...pageUrls];
-  for (const subUrl of subSitemapUrls) {
-    if (collected.length >= MAX_URLS) break;
-    collected.push(...await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher));
+  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+    );
+    for (const sub of subResults) collected.push(...sub);
   }
   return [...new Set(collected)].slice(0, MAX_URLS);
 }

package/dist/sitemap.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/sitemap.ts"],
-  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  for (const subUrl of subSitemapUrls) {\n    if (collected.length >= MAX_URLS) break;\n    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
-  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,aAAW,UAAU,gBAAgB;AACnC,QAAI,UAAU,UAAU,SAAU;AAClC,cAAU,KAAK,GAAI,MAAM,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAE;AAAA,EACnF;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
+  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {\n    const subResults = await Promise.all(\n      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))\n    );\n    for (const sub of subResults) collected.push(...sub);\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
+  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,MAAI,eAAe,SAAS,KAAK,UAAU,SAAS,UAAU;AAC5D,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,eAAe,MAAM,GAAG,EAAE,EAAE,IAAI,YAAU,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAC;AAAA,IACnG;AACA,eAAW,OAAO,WAAY,WAAU,KAAK,GAAG,GAAG;AAAA,EACrD;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
   "names": []
 }

package/dist/url-pattern-completer.js

@@ -47,6 +47,7 @@ Respond with ONLY valid JSON, no explanation:
     const response = await client.messages.create({
       model: llmCfg.model ?? "claude-haiku-4-5-20251001",
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore: cache_control supported in beta
       system: [{ type: "text", text: "You are a URL pattern analyst. Always respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
       messages: [{ role: "user", content: prompt }]

package/dist/url-pattern-completer.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/url-pattern-completer.ts"],
-  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
-  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
+  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      temperature: 0,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
+  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA,MACZ,aAAa;AAAA;AAAA,MAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.6",
+  "version": "1.1.7",
   "private": false,
   "publishConfig": {
     "access": "public",