@detiq/crawler 0.1.3

package/dist/sitemap.js

@@ -0,0 +1,152 @@
+import { XMLParser } from "fast-xml-parser";
+import { ProxyAgent } from "undici";
+import { assertCrawlUrlSafe } from "@qip/core";
+import { logger } from "./logger.js";
+const FETCH_TIMEOUT_MS = 1e4;
+const MAX_SITEMAP_INDEX_DEPTH = 3;
+const MAX_SUB_SITEMAPS = 50;
+const MAX_URLS = 5e3;
+function toArray(x) {
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
+}
+function hostKey(hostname) {
+  return hostname.replace(/^www\./i, "").toLowerCase();
+}
+function sameSite(candidateUrl, siteOrigin) {
+  try {
+    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
+    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
+  } catch {
+    return false;
+  }
+}
+function extractUrlsFromParsedSitemap(parsed, origin) {
+  const rawPageUrls = [];
+  let subSitemapUrls = [];
+  if (parsed?.sitemapindex) {
+    subSitemapUrls = toArray(parsed.sitemapindex.sitemap).map((s) => s?.loc).filter((loc) => typeof loc === "string").slice(0, MAX_SUB_SITEMAPS);
+  }
+  if (parsed?.urlset) {
+    rawPageUrls.push(
+      ...toArray(parsed.urlset.url).map((u) => u?.loc).filter((loc) => typeof loc === "string")
+    );
+  }
+  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+}
+async function fetchAndParseXml(url, dispatcher) {
+  try {
+    assertCrawlUrlSafe(url);
+  } catch {
+    return null;
+  }
+  try {
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+sitemap-discovery)" },
+      ...dispatcher ? { dispatcher } : {}
+    });
+    if (!res.ok) return null;
+    const text = await res.text();
+    return new XMLParser().parse(text);
+  } catch {
+    return null;
+  }
+}
+async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
+  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
+  if (!parsed) return [];
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+  const collected = [...pageUrls];
+  for (const subUrl of subSitemapUrls) {
+    if (collected.length >= MAX_URLS) break;
+    collected.push(...await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher));
+  }
+  return [...new Set(collected)].slice(0, MAX_URLS);
+}
+async function discoverSitemapUrls(appUrl, sitemapUrl) {
+  let origin;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return [];
+  }
+  const target = sitemapUrl || `${origin}/sitemap.xml`;
+  try {
+    return await fetchSitemapUrls(target, origin);
+  } catch (err) {
+    logger.warn({ err: String(err), target }, "[sitemap] discovery failed, continuing without it");
+    return [];
+  }
+}
+function buildProxyDispatcher(proxyUrl) {
+  if (!proxyUrl || proxyUrl === "auto") return void 0;
+  try {
+    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (!m) return new ProxyAgent(proxyUrl);
+    const [, scheme, user, pass, hostPort] = m;
+    const uri = `${scheme}://${hostPort}`;
+    if (user) {
+      const token = "Basic " + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? "")}`).toString("base64");
+      return new ProxyAgent({ uri, token });
+    }
+    return new ProxyAgent({ uri });
+  } catch {
+    return void 0;
+  }
+}
+function parseRobotsSitemaps(robotsTxt, origin) {
+  const out = [];
+  for (const line of robotsTxt.split(/\r?\n/)) {
+    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
+    if (m && sameSite(m[1], origin)) out.push(m[1]);
+  }
+  return [...new Set(out)];
+}
+async function discoverProjectUrls(appUrl, opts) {
+  let origin;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return { candidates: [], sitemapFound: false };
+  }
+  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  let robotsSitemaps = [];
+  try {
+    assertCrawlUrlSafe(`${origin}/robots.txt`);
+    const res = await fetch(`${origin}/robots.txt`, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+url-discovery)" },
+      ...dispatcher ? { dispatcher } : {}
+    });
+    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
+  } catch {
+  }
+  const robotsSet = new Set(robotsSitemaps);
+  const sitemapTargets = [.../* @__PURE__ */ new Set([
+    ...opts?.sitemapUrl ? [opts.sitemapUrl] : [],
+    ...robotsSitemaps,
+    `${origin}/sitemap.xml`
+  ])];
+  const seen = /* @__PURE__ */ new Map();
+  let sitemapFound = false;
+  for (const sm of sitemapTargets) {
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+    if (urls.length) sitemapFound = true;
+    const source = robotsSet.has(sm) ? "robots" : "sitemap";
+    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
+  }
+  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+}
+export {
+  buildProxyDispatcher,
+  discoverProjectUrls,
+  discoverSitemapUrls,
+  extractUrlsFromParsedSitemap,
+  fetchSitemapUrls,
+  parseRobotsSitemaps,
+  sameSite
+};
+//# sourceMappingURL=sitemap.js.map

package/dist/sitemap.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/sitemap.ts"],
+  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@qip/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  for (const subUrl of subSitemapUrls) {\n    if (collected.length >= MAX_URLS) break;\n    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
+  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,aAAW,UAAU,gBAAgB;AACnC,QAAI,UAAU,UAAU,SAAU;AAClC,cAAU,KAAK,GAAI,MAAM,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAE;AAAA,EACnF;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
+  "names": []
+}

package/dist/skyvern.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Skyvern Vision Fallback
+ *
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
+ * government portals. It's the "break glass" option when Stagehand fails.
+ *
+ * Production setup (Phase 4+):
+ *   1. Self-host: https://github.com/Skyvern-AI/skyvern
+ *   2. Or use cloud: https://app.skyvern.com
+ *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
+ *
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
+ * Use only when the fast path fails.
+ */
+export interface SkyvernTask {
+    taskDescription: string;
+    url: string;
+    credentials?: {
+        username: string;
+        password: string;
+        oneTimeCode?: string;
+    };
+}
+export interface SkyvernConfig {
+    apiUrl: string;
+    apiKey: string;
+    llmKey?: string;
+    llmApiKey?: string;
+}
+export interface SkyvernStep {
+    order: number;
+    action_type: string;
+    element_description?: string;
+    reasoning?: string;
+    success: boolean;
+    screenshot_url?: string;
+    duration_ms?: number;
+    error?: string;
+}
+export interface SkyvernNavigateResult {
+    screenshotUrls: string[];
+    visitedUrls: string[];
+    status: 'success' | 'failed' | 'partial';
+    steps?: SkyvernStep[];
+    taskId?: string;
+    durationMs?: number;
+    triggerReason?: string;
+}
+/**
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
+ */
+export declare function skyvernNavigate(startUrl: string, navigationGoals: string[], tenantConfig?: SkyvernConfig, triggerReason?: string): Promise<SkyvernNavigateResult>;
+export declare function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'>;

package/dist/skyvern.js

@@ -0,0 +1,129 @@
+async function skyvernNavigate(startUrl, navigationGoals, tenantConfig, triggerReason) {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+  if (!apiUrl || !apiKey) {
+    console.warn("[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation");
+    return { screenshotUrls: [], visitedUrls: [], status: "failed" };
+  }
+  const visitedUrls = [];
+  const screenshotUrls = [];
+  const skyvernSteps = [];
+  let lastTaskId;
+  let totalDurationMs = 0;
+  for (const goal of navigationGoals) {
+    try {
+      const taskStart = Date.now();
+      const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "x-api-key": apiKey },
+        body: JSON.stringify({
+          url: startUrl,
+          navigation_goal: goal,
+          ...tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {},
+          ...tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}
+        })
+      });
+      if (!res.ok) continue;
+      const data = await res.json();
+      const taskId = data.task_id;
+      if (!taskId) continue;
+      lastTaskId = taskId;
+      let done = false;
+      for (let p = 0; p < 12 && !done; p++) {
+        await new Promise((r) => setTimeout(r, 5e3));
+        const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
+          headers: { "x-api-key": apiKey }
+        }).catch(() => null);
+        if (!poll?.ok) continue;
+        const pollData = await poll.json();
+        if (pollData.status === "completed" || pollData.status === "failed") {
+          done = true;
+          totalDurationMs += Date.now() - taskStart;
+          if (pollData.status === "completed") {
+            if (pollData.output?.url) visitedUrls.push(pollData.output.url);
+            if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);
+          }
+          try {
+            const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
+              headers: { "x-api-key": apiKey }
+            }).catch(() => null);
+            if (stepsRes?.ok) {
+              const stepsData = await stepsRes.json();
+              const stepsList = Array.isArray(stepsData) ? stepsData : stepsData?.steps ?? [];
+              for (const s of stepsList) {
+                if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);
+                skyvernSteps.push({
+                  order: s.step_order ?? s.order ?? 0,
+                  action_type: s.action_type ?? s.action ?? "unknown",
+                  element_description: s.output?.element_description,
+                  reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
+                  success: s.status === "completed",
+                  screenshot_url: s.output?.screenshot_url,
+                  error: s.status === "failed" ? s.output?.errors?.[0] ?? "Step failed" : void 0
+                });
+              }
+            }
+          } catch {
+          }
+        }
+      }
+    } catch (err) {
+      console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
+    }
+  }
+  return {
+    screenshotUrls,
+    visitedUrls,
+    status: visitedUrls.length > 0 ? "success" : screenshotUrls.length > 0 ? "partial" : "failed",
+    steps: skyvernSteps,
+    taskId: lastTaskId,
+    durationMs: totalDurationMs || void 0,
+    triggerReason
+  };
+}
+async function skyvernFallback(task, tenantConfig) {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+  if (!apiUrl || !apiKey) {
+    console.warn(
+      "[skyvern] Vision fallback disabled \u2014 set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.",
+      "\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern"
+    );
+    return "failed";
+  }
+  try {
+    const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "x-api-key": apiKey },
+      body: JSON.stringify({
+        url: task.url,
+        navigation_goal: task.taskDescription,
+        ...task.credentials && {
+          navigation_payload: {
+            username: task.credentials.username,
+            password: task.credentials.password,
+            ...task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }
+          }
+        },
+        ...tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {},
+        ...tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}
+      })
+    });
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      console.error(`[skyvern] Task creation failed: ${res.status}`, body);
+      return "failed";
+    }
+    const data = await res.json();
+    console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
+    return "success";
+  } catch (err) {
+    console.error("[skyvern] Error:", err);
+    return "failed";
+  }
+}
+export {
+  skyvernFallback,
+  skyvernNavigate
+};
+//# sourceMappingURL=skyvern.js.map

package/dist/skyvern.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/skyvern.ts"],
+  "sourcesContent": ["/**\n * Skyvern Vision Fallback\n *\n * Skyvern combines computer vision + LLM reasoning to navigate sites that\n * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,\n * government portals. It's the \"break glass\" option when Stagehand fails.\n *\n * Production setup (Phase 4+):\n *   1. Self-host: https://github.com/Skyvern-AI/skyvern\n *   2. Or use cloud: https://app.skyvern.com\n *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env\n *\n * Pricing note: vision tasks cost more per step than DOM-based Stagehand.\n * Use only when the fast path fails.\n */\n\nexport interface SkyvernTask {\n  taskDescription: string;\n  url: string;\n  credentials?: { username: string; password: string; oneTimeCode?: string };\n}\n\nexport interface SkyvernConfig {\n  apiUrl: string;\n  apiKey: string;\n  llmKey?: string;\n  llmApiKey?: string;\n}\n\nexport interface SkyvernStep {\n  order: number;\n  action_type: string;\n  element_description?: string;\n  reasoning?: string;\n  success: boolean;\n  screenshot_url?: string;\n  duration_ms?: number;\n  error?: string;\n}\n\nexport interface SkyvernNavigateResult {\n  screenshotUrls: string[];\n  visitedUrls: string[];\n  status: 'success' | 'failed' | 'partial';\n  steps?: SkyvernStep[];\n  taskId?: string;\n  durationMs?: number;\n  triggerReason?: string;\n}\n\n/**\n * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis\n * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).\n * Skyvern drives the browser via screenshot + LLM, returning visited page list.\n */\nexport async function skyvernNavigate(\n  startUrl: string,\n  navigationGoals: string[],\n  tenantConfig?: SkyvernConfig,\n  triggerReason?: string,\n): Promise<SkyvernNavigateResult> {\n  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;\n  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;\n\n  if (!apiUrl || !apiKey) {\n    console.warn('[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation');\n    return { screenshotUrls: [], visitedUrls: [], status: 'failed' };\n  }\n\n  const visitedUrls: string[] = [];\n  const screenshotUrls: string[] = [];\n  const skyvernSteps: SkyvernStep[] = [];\n  let lastTaskId: string | undefined;\n  let totalDurationMs = 0;\n\n  for (const goal of navigationGoals) {\n    try {\n      const taskStart = Date.now();\n      const res = await fetch(`${apiUrl}/api/v1/tasks`, {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },\n        body: JSON.stringify({\n          url: startUrl,\n          navigation_goal: goal,\n          ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),\n          ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),\n        }),\n      });\n      if (!res.ok) continue;\n      const data: any = await res.json();\n      const taskId: string = data.task_id;\n      if (!taskId) continue;\n      lastTaskId = taskId;\n\n      // Poll for task completion (max 60s per goal)\n      let done = false;\n      for (let p = 0; p < 12 && !done; p++) {\n        await new Promise((r) => setTimeout(r, 5000));\n        const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {\n          headers: { 'x-api-key': apiKey },\n        }).catch(() => null);\n        if (!poll?.ok) continue;\n        const pollData: any = await poll.json();\n        if (pollData.status === 'completed' || pollData.status === 'failed') {\n          done = true;\n          totalDurationMs += Date.now() - taskStart;\n          if (pollData.status === 'completed') {\n            if (pollData.output?.url) visitedUrls.push(pollData.output.url);\n            if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);\n          }\n\n          // Fetch step-level detail from Skyvern\n          try {\n            const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {\n              headers: { 'x-api-key': apiKey },\n            }).catch(() => null);\n            if (stepsRes?.ok) {\n              const stepsData: any = await stepsRes.json();\n              const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);\n              for (const s of stepsList) {\n                if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);\n                skyvernSteps.push({\n                  order: s.step_order ?? s.order ?? 0,\n                  action_type: s.action_type ?? s.action ?? 'unknown',\n                  element_description: s.output?.element_description,\n                  reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,\n                  success: s.status === 'completed',\n                  screenshot_url: s.output?.screenshot_url,\n                  error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,\n                });\n              }\n            }\n          } catch { /* non-fatal */ }\n        }\n      }\n    } catch (err) {\n      console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);\n    }\n  }\n\n  return {\n    screenshotUrls,\n    visitedUrls,\n    status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',\n    steps: skyvernSteps,\n    taskId: lastTaskId,\n    durationMs: totalDurationMs || undefined,\n    triggerReason,\n  };\n}\n\nexport async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'> {\n  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;\n  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;\n\n  if (!apiUrl || !apiKey) {\n    console.warn(\n      '[skyvern] Vision fallback disabled \u2014 set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.',\n      '\\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern'\n    );\n    return 'failed';\n  }\n\n  try {\n    const res = await fetch(`${apiUrl}/api/v1/tasks`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },\n      body: JSON.stringify({\n        url: task.url,\n        navigation_goal: task.taskDescription,\n        ...(task.credentials && {\n          navigation_payload: {\n            username: task.credentials.username,\n            password: task.credentials.password,\n            ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),\n          },\n        }),\n        ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),\n        ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),\n      }),\n    });\n\n    if (!res.ok) {\n      const body = await res.text().catch(() => '');\n      console.error(`[skyvern] Task creation failed: ${res.status}`, body);\n      return 'failed';\n    }\n\n    const data: any = await res.json();\n    console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);\n    // Full implementation: poll until status === \"completed\" | \"failed\".\n    // For Phase 1 the login already happened in the browser context Skyvern manages,\n    // so returning 'success' here means we can proceed in that browser session.\n    return 'success';\n  } catch (err) {\n    console.error('[skyvern] Error:', err);\n    return 'failed';\n  }\n}\n"],
+  "mappings": "AAuDA,eAAsB,gBACpB,UACA,iBACA,cACA,eACgC;AAChC,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AACnD,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AAEnD,MAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAQ,KAAK,uFAAkF;AAC/F,WAAO,EAAE,gBAAgB,CAAC,GAAG,aAAa,CAAC,GAAG,QAAQ,SAAS;AAAA,EACjE;AAEA,QAAM,cAAwB,CAAC;AAC/B,QAAM,iBAA2B,CAAC;AAClC,QAAM,eAA8B,CAAC;AACrC,MAAI;AACJ,MAAI,kBAAkB;AAEtB,aAAW,QAAQ,iBAAiB;AAClC,QAAI;AACF,YAAM,YAAY,KAAK,IAAI;AAC3B,YAAM,MAAM,MAAM,MAAM,GAAG,MAAM,iBAAiB;AAAA,QAChD,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,oBAAoB,aAAa,OAAO;AAAA,QACnE,MAAM,KAAK,UAAU;AAAA,UACnB,KAAK;AAAA,UACL,iBAAiB;AAAA,UACjB,GAAI,cAAc,SAAS,EAAE,SAAS,aAAa,OAAO,IAAI,QAAQ,IAAI,kBAAkB,EAAE,SAAS,QAAQ,IAAI,gBAAgB,IAAI,CAAC;AAAA,UACxI,GAAI,cAAc,YAAY,EAAE,aAAa,aAAa,UAAU,IAAI,QAAQ,IAAI,iBAAiB,EAAE,aAAa,QAAQ,IAAI,eAAe,IAAI,CAAC;AAAA,QACtJ,CAAC;AAAA,MACH,CAAC;AACD,UAAI,CAAC,IAAI,GAAI;AACb,YAAM,OAAY,MAAM,IAAI,KAAK;AACjC,YAAM,SAAiB,KAAK;AAC5B,UAAI,CAAC,OAAQ;AACb,mBAAa;AAGb,UAAI,OAAO;AACX,eAAS,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK;AACpC,cAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAI,CAAC;AAC5C,cAAM,OAAO,MAAM,MAAM,GAAG,MAAM,iBAAiB,MAAM,IAAI;AAAA,UAC3D,SAAS,EAAE,aAAa,OAAO;AAAA,QACjC,CAAC,EAAE,MAAM,MAAM,IAAI;AACnB,YAAI,CAAC,MAAM,GAAI;AACf,cAAM,WAAgB,MAAM,KAAK,KAAK;AACtC,YAAI,SAAS,WAAW,eAAe,SAAS,WAAW,UAAU;AACnE,iBAAO;AACP,6BAAmB,KAAK,IAAI,IAAI;AAChC,cAAI,SAAS,WAAW,aAAa;AACnC,gBAAI,SAAS,QAAQ,IAAK,aAAY,KAAK,SAAS,OAAO,GAAG;AAC9D,gBAAI,SAAS,QAAQ,eAAgB,gBAAe,KAAK,SAAS,OAAO,cAAc;AAAA,UACzF;AAGA,cAAI;AACF,kBAAM,WAAW,MAAM,MAAM,GAAG,MAAM,iBAAiB,MAAM,UAAU;AAAA,cACrE,SAAS,EAAE,aAAa,OAAO;AAAA,YACjC,CAAC,EAAE,MAAM,MAAM,IAAI;AACnB,gBAAI,UAAU,IAAI;AAChB,oBAAM,YAAiB,MAAM,SAAS,KAAK;AAC3C,oBAAM,YAAY,MAAM,QAAQ,SAAS,IAAI,YAAa,WAAW,SAAS,CAAC;AAC/E,yBAAW,KAAK,WAAW;AACzB,oBAAI,EAAE,QAAQ,eAAgB,gBAAe,KAAK,EAAE,OAAO,cAAc;AACzE,6BAAa,KAAK;AAAA,kBAChB,OAAO,EAAE,cAAc,EAAE,SAAS;AAAA,kBAClC,aAAa,EAAE,eAAe,EAAE,UAAU;AAAA,kBAC1C,qBAAqB,EAAE,QAAQ;AAAA,kBAC/B,WAAW,EAAE,QAAQ,cAAc,sBAAsB,EAAE,QAAQ;AAAA,kBACnE,SAAS,EAAE,WAAW;AAAA,kBACtB,gBAAgB,EAAE,QAAQ;AAAA,kBAC1B,OAAO,EAAE,WAAW,WAAY,EAAE,QAAQ,SAAS,CAAC,KAAK,gBAAiB;AAAA,gBAC5E,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF,QAAQ;AAAA,UAAkB;AAAA,QAC5B;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,cAAQ,KAAK,qCAAqC,IAAI,IAAI,GAAG;AAAA,IAC/D;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,YAAY,SAAS,IAAI,YAAY,eAAe,SAAS,IAAI,YAAY;AAAA,IACrF,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,YAAY,mBAAmB;AAAA,IAC/B;AAAA,EACF;AACF;AAEA,eAAsB,gBAAgB,MAAmB,cAA6D;AACpH,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AACnD,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AAEnD,MAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAQ;AAAA,MACN;AAAA,MACA;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,iBAAiB;AAAA,MAChD,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,oBAAoB,aAAa,OAAO;AAAA,MACnE,MAAM,KAAK,UAAU;AAAA,QACnB,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,GAAI,KAAK,eAAe;AAAA,UACtB,oBAAoB;AAAA,YAClB,UAAU,KAAK,YAAY;AAAA,YAC3B,UAAU,KAAK,YAAY;AAAA,YAC3B,GAAI,KAAK,YAAY,eAAe,EAAE,KAAK,KAAK,YAAY,YAAY;AAAA,UAC1E;AAAA,QACF;AAAA,QACA,GAAI,cAAc,SAAS,EAAE,SAAS,aAAa,OAAO,IAAI,QAAQ,IAAI,kBAAkB,EAAE,SAAS,QAAQ,IAAI,gBAAgB,IAAI,CAAC;AAAA,QACxI,GAAI,cAAc,YAAY,EAAE,aAAa,aAAa,UAAU,IAAI,QAAQ,IAAI,iBAAiB,EAAE,aAAa,QAAQ,IAAI,eAAe,IAAI,CAAC;AAAA,MACtJ,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,cAAQ,MAAM,mCAAmC,IAAI,MAAM,IAAI,IAAI;AACnE,aAAO;AAAA,IACT;AAEA,UAAM,OAAY,MAAM,IAAI,KAAK;AACjC,YAAQ,IAAI,0BAA0B,KAAK,OAAO,4BAA4B,KAAK,OAAO,kBAAkB;AAI5G,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,YAAQ,MAAM,oBAAoB,GAAG;AACrC,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/sqs-queue.d.ts

@@ -0,0 +1,2 @@
+export declare function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void>;
+export declare function startSqsWorker(): Promise<void>;

package/dist/sqs-queue.js

@@ -0,0 +1,47 @@
+import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from "@aws-sdk/client-sqs";
+const sqs = new SQSClient({ region: process.env.AWS_REGION ?? "us-east-1" });
+async function sendJobToSqs(jobId, tenantId, config) {
+  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
+  await sqs.send(new SendMessageCommand({
+    QueueUrl: queueUrl,
+    MessageBody: JSON.stringify({ jobId, tenantId, config })
+  }));
+}
+async function startSqsWorker() {
+  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
+  const { processJob } = await import("./job-queue.js");
+  console.log("[sqs-worker] starting, queue:", queueUrl);
+  while (true) {
+    try {
+      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+        QueueUrl: queueUrl,
+        WaitTimeSeconds: 20,
+        MaxNumberOfMessages: 2
+      }));
+      await Promise.all(Messages.map(async (msg) => {
+        let jobId = "?";
+        try {
+          const data = JSON.parse(msg.Body);
+          jobId = data.jobId;
+          await processJob(data.jobId, data.config);
+          await sqs.send(new DeleteMessageCommand({
+            QueueUrl: queueUrl,
+            ReceiptHandle: msg.ReceiptHandle
+          }));
+        } catch (err) {
+          console.error(`[sqs-worker] job ${jobId} failed \u2014 leaving for SQS redrive:`, err.message);
+        }
+      }));
+    } catch (err) {
+      console.error("[sqs-worker] receive error:", err.message);
+      await new Promise((r) => setTimeout(r, 2e3));
+    }
+  }
+}
+export {
+  sendJobToSqs,
+  startSqsWorker
+};
+//# sourceMappingURL=sqs-queue.js.map

package/dist/sqs-queue.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/sqs-queue.ts"],
+  "sourcesContent": ["import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';\n\nconst sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });\n\nexport async function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void> {\n  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;\n  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');\n  await sqs.send(new SendMessageCommand({\n    QueueUrl: queueUrl,\n    MessageBody: JSON.stringify({ jobId, tenantId, config }),\n  }));\n}\n\nexport async function startSqsWorker(): Promise<void> {\n  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;\n  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');\n\n  const { processJob } = await import('./job-queue.js');\n  console.log('[sqs-worker] starting, queue:', queueUrl);\n\n  while (true) {\n    try {\n      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({\n        QueueUrl: queueUrl,\n        WaitTimeSeconds: 20,\n        MaxNumberOfMessages: 2,\n      }));\n\n      await Promise.all(Messages.map(async (msg) => {\n        let jobId = '?';\n        try {\n          const data = JSON.parse(msg.Body!);\n          jobId = data.jobId;\n          await processJob(data.jobId, data.config);\n          await sqs.send(new DeleteMessageCommand({\n            QueueUrl: queueUrl,\n            ReceiptHandle: msg.ReceiptHandle!,\n          }));\n        } catch (err: any) {\n          console.error(`[sqs-worker] job ${jobId} failed \u2014 leaving for SQS redrive:`, err.message);\n          // Do NOT delete \u2014 SQS redrive policy + DLQ handles retry\n        }\n      }));\n    } catch (err: any) {\n      console.error('[sqs-worker] receive error:', err.message);\n      await new Promise((r) => setTimeout(r, 2000));\n    }\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,WAAW,oBAAoB,uBAAuB,4BAA4B;AAE3F,MAAM,MAAM,IAAI,UAAU,EAAE,QAAQ,QAAQ,IAAI,cAAc,YAAY,CAAC;AAE3E,eAAsB,aAAa,OAAe,UAAkB,QAA4B;AAC9F,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,oCAAoC;AACnE,QAAM,IAAI,KAAK,IAAI,mBAAmB;AAAA,IACpC,UAAU;AAAA,IACV,aAAa,KAAK,UAAU,EAAE,OAAO,UAAU,OAAO,CAAC;AAAA,EACzD,CAAC,CAAC;AACJ;AAEA,eAAsB,iBAAgC;AACpD,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,oCAAoC;AAEnE,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,gBAAgB;AACpD,UAAQ,IAAI,iCAAiC,QAAQ;AAErD,SAAO,MAAM;AACX,QAAI;AACF,YAAM,EAAE,WAAW,CAAC,EAAE,IAAI,MAAM,IAAI,KAAK,IAAI,sBAAsB;AAAA,QACjE,UAAU;AAAA,QACV,iBAAiB;AAAA,QACjB,qBAAqB;AAAA,MACvB,CAAC,CAAC;AAEF,YAAM,QAAQ,IAAI,SAAS,IAAI,OAAO,QAAQ;AAC5C,YAAI,QAAQ;AACZ,YAAI;AACF,gBAAM,OAAO,KAAK,MAAM,IAAI,IAAK;AACjC,kBAAQ,KAAK;AACb,gBAAM,WAAW,KAAK,OAAO,KAAK,MAAM;AACxC,gBAAM,IAAI,KAAK,IAAI,qBAAqB;AAAA,YACtC,UAAU;AAAA,YACV,eAAe,IAAI;AAAA,UACrB,CAAC,CAAC;AAAA,QACJ,SAAS,KAAU;AACjB,kBAAQ,MAAM,oBAAoB,KAAK,2CAAsC,IAAI,OAAO;AAAA,QAE1F;AAAA,MACF,CAAC,CAAC;AAAA,IACJ,SAAS,KAAU;AACjB,cAAQ,MAAM,+BAA+B,IAAI,OAAO;AACxD,YAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAI,CAAC;AAAA,IAC9C;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/sso-token.d.ts

@@ -0,0 +1,29 @@
+/**
+ * SSO token retrieval for Okta and Azure AD.
+ * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
+ * without browser-based login.
+ *
+ * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
+ * for service accounts and automated testing scenarios.
+ */
+export type SsoProvider = 'okta' | 'azure_ad' | 'generic_oidc';
+export interface SsoConfig {
+    provider: SsoProvider;
+    /** Okta domain (e.g. 'mycompany.okta.com') or Azure tenant ID */
+    domain: string;
+    /** OAuth2 client ID */
+    clientId: string;
+    /** OAuth2 client secret (for client_credentials or confidential ROPC) */
+    clientSecret?: string;
+    /** Scopes to request (space-separated). Default: 'openid profile' */
+    scope?: string;
+}
+/**
+ * Get an access token using ROPC flow (username + password → token).
+ * Returns the access token string.
+ */
+export declare function getSsoToken(config: SsoConfig, username: string, password: string): Promise<string>;
+/**
+ * Get a client credentials token (no user — for service accounts).
+ */
+export declare function getSsoClientCredentialsToken(config: SsoConfig): Promise<string>;

package/dist/sso-token.js

@@ -0,0 +1,76 @@
+async function getSsoToken(config, username, password) {
+  let tokenUrl;
+  const params = {
+    grant_type: "password",
+    username,
+    password,
+    scope: config.scope ?? "openid profile",
+    client_id: config.clientId
+  };
+  if (config.clientSecret) {
+    params.client_secret = config.clientSecret;
+  }
+  switch (config.provider) {
+    case "okta":
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case "azure_ad":
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    case "generic_oidc":
+      tokenUrl = config.domain.startsWith("http") ? config.domain : `https://${config.domain}/oauth2/token`;
+      break;
+    default:
+      throw new Error(`Unknown SSO provider: ${config.provider}`);
+  }
+  const res = await fetch(tokenUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams(params).toString()
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO token request failed (${res.status}): ${body}`);
+  }
+  const data = await res.json();
+  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);
+  if (!data.access_token) throw new Error("SSO response missing access_token");
+  return data.access_token;
+}
+async function getSsoClientCredentialsToken(config) {
+  if (!config.clientSecret) throw new Error("clientSecret required for client credentials flow");
+  let tokenUrl;
+  switch (config.provider) {
+    case "okta":
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case "azure_ad":
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    default:
+      tokenUrl = `https://${config.domain}/oauth2/token`;
+  }
+  const res = await fetch(tokenUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      grant_type: "client_credentials",
+      client_id: config.clientId,
+      client_secret: config.clientSecret,
+      scope: config.scope ?? "openid profile"
+    }).toString()
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
+  }
+  const data = await res.json();
+  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);
+  if (!data.access_token) throw new Error("SSO response missing access_token");
+  return data.access_token;
+}
+export {
+  getSsoClientCredentialsToken,
+  getSsoToken
+};
+//# sourceMappingURL=sso-token.js.map

package/dist/sso-token.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/sso-token.ts"],
+  "sourcesContent": ["/**\n * SSO token retrieval for Okta and Azure AD.\n * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens\n * without browser-based login.\n *\n * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD\n * for service accounts and automated testing scenarios.\n */\n\nexport type SsoProvider = 'okta' | 'azure_ad' | 'generic_oidc';\n\nexport interface SsoConfig {\n  provider: SsoProvider;\n  /** Okta domain (e.g. 'mycompany.okta.com') or Azure tenant ID */\n  domain: string;\n  /** OAuth2 client ID */\n  clientId: string;\n  /** OAuth2 client secret (for client_credentials or confidential ROPC) */\n  clientSecret?: string;\n  /** Scopes to request (space-separated). Default: 'openid profile' */\n  scope?: string;\n}\n\n/**\n * Get an access token using ROPC flow (username + password \u2192 token).\n * Returns the access token string.\n */\nexport async function getSsoToken(\n  config: SsoConfig,\n  username: string,\n  password: string\n): Promise<string> {\n  let tokenUrl: string;\n  const params: Record<string, string> = {\n    grant_type: 'password',\n    username,\n    password,\n    scope: config.scope ?? 'openid profile',\n    client_id: config.clientId,\n  };\n\n  if (config.clientSecret) {\n    params.client_secret = config.clientSecret;\n  }\n\n  switch (config.provider) {\n    case 'okta':\n      tokenUrl = `https://${config.domain}/oauth2/v1/token`;\n      break;\n    case 'azure_ad':\n      // domain here is the tenant ID (e.g. XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX or 'mycompany.onmicrosoft.com')\n      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;\n      break;\n    case 'generic_oidc':\n      tokenUrl = config.domain.startsWith('http') ? config.domain : `https://${config.domain}/oauth2/token`;\n      break;\n    default:\n      throw new Error(`Unknown SSO provider: ${config.provider}`);\n  }\n\n  const res = await fetch(tokenUrl, {\n    method: 'POST',\n    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n    body: new URLSearchParams(params).toString(),\n  });\n\n  if (!res.ok) {\n    const body = await res.text().catch(() => res.statusText);\n    throw new Error(`SSO token request failed (${res.status}): ${body}`);\n  }\n\n  const data = await res.json() as { access_token?: string; error?: string; error_description?: string };\n  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);\n  if (!data.access_token) throw new Error('SSO response missing access_token');\n\n  return data.access_token;\n}\n\n/**\n * Get a client credentials token (no user \u2014 for service accounts).\n */\nexport async function getSsoClientCredentialsToken(\n  config: SsoConfig\n): Promise<string> {\n  if (!config.clientSecret) throw new Error('clientSecret required for client credentials flow');\n\n  let tokenUrl: string;\n  switch (config.provider) {\n    case 'okta':\n      tokenUrl = `https://${config.domain}/oauth2/v1/token`;\n      break;\n    case 'azure_ad':\n      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;\n      break;\n    default:\n      tokenUrl = `https://${config.domain}/oauth2/token`;\n  }\n\n  const res = await fetch(tokenUrl, {\n    method: 'POST',\n    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n    body: new URLSearchParams({\n      grant_type: 'client_credentials',\n      client_id: config.clientId,\n      client_secret: config.clientSecret,\n      scope: config.scope ?? 'openid profile',\n    }).toString(),\n  });\n\n  if (!res.ok) {\n    const body = await res.text().catch(() => res.statusText);\n    throw new Error(`SSO client credentials failed (${res.status}): ${body}`);\n  }\n\n  const data = await res.json() as { access_token?: string; error?: string; error_description?: string };\n  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);\n  if (!data.access_token) throw new Error('SSO response missing access_token');\n\n  return data.access_token;\n}\n"],
+  "mappings": "AA2BA,eAAsB,YACpB,QACA,UACA,UACiB;AACjB,MAAI;AACJ,QAAM,SAAiC;AAAA,IACrC,YAAY;AAAA,IACZ;AAAA,IACA;AAAA,IACA,OAAO,OAAO,SAAS;AAAA,IACvB,WAAW,OAAO;AAAA,EACpB;AAEA,MAAI,OAAO,cAAc;AACvB,WAAO,gBAAgB,OAAO;AAAA,EAChC;AAEA,UAAQ,OAAO,UAAU;AAAA,IACvB,KAAK;AACH,iBAAW,WAAW,OAAO,MAAM;AACnC;AAAA,IACF,KAAK;AAEH,iBAAW,qCAAqC,OAAO,MAAM;AAC7D;AAAA,IACF,KAAK;AACH,iBAAW,OAAO,OAAO,WAAW,MAAM,IAAI,OAAO,SAAS,WAAW,OAAO,MAAM;AACtF;AAAA,IACF;AACE,YAAM,IAAI,MAAM,yBAAyB,OAAO,QAAQ,EAAE;AAAA,EAC9D;AAEA,QAAM,MAAM,MAAM,MAAM,UAAU;AAAA,IAChC,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oCAAoC;AAAA,IAC/D,MAAM,IAAI,gBAAgB,MAAM,EAAE,SAAS;AAAA,EAC7C,CAAC;AAED,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,UAAU;AACxD,UAAM,IAAI,MAAM,6BAA6B,IAAI,MAAM,MAAM,IAAI,EAAE;AAAA,EACrE;AAEA,QAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,MAAI,KAAK,MAAO,OAAM,IAAI,MAAM,cAAc,KAAK,KAAK,WAAM,KAAK,iBAAiB,EAAE;AACtF,MAAI,CAAC,KAAK,aAAc,OAAM,IAAI,MAAM,mCAAmC;AAE3E,SAAO,KAAK;AACd;AAKA,eAAsB,6BACpB,QACiB;AACjB,MAAI,CAAC,OAAO,aAAc,OAAM,IAAI,MAAM,mDAAmD;AAE7F,MAAI;AACJ,UAAQ,OAAO,UAAU;AAAA,IACvB,KAAK;AACH,iBAAW,WAAW,OAAO,MAAM;AACnC;AAAA,IACF,KAAK;AACH,iBAAW,qCAAqC,OAAO,MAAM;AAC7D;AAAA,IACF;AACE,iBAAW,WAAW,OAAO,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,MAAM,MAAM,UAAU;AAAA,IAChC,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oCAAoC;AAAA,IAC/D,MAAM,IAAI,gBAAgB;AAAA,MACxB,YAAY;AAAA,MACZ,WAAW,OAAO;AAAA,MAClB,eAAe,OAAO;AAAA,MACtB,OAAO,OAAO,SAAS;AAAA,IACzB,CAAC,EAAE,SAAS;AAAA,EACd,CAAC;AAED,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,UAAU;AACxD,UAAM,IAAI,MAAM,kCAAkC,IAAI,MAAM,MAAM,IAAI,EAAE;AAAA,EAC1E;AAEA,QAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,MAAI,KAAK,MAAO,OAAM,IAAI,MAAM,cAAc,KAAK,KAAK,WAAM,KAAK,iBAAiB,EAAE;AACtF,MAAI,CAAC,KAAK,aAAc,OAAM,IAAI,MAAM,mCAAmC;AAE3E,SAAO,KAAK;AACd;",
+  "names": []
+}

package/dist/stealth/human-behavior.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright';
+export declare function humanDelay(min?: number, max?: number): Promise<void>;
+export declare function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void>;
+export declare function simulateMouseHover(page: Page, selector: string): Promise<void>;
+export declare function simulateClick(page: Page, selector: string): Promise<void>;
+export declare function injectFingerprintOverrides(page: Page): Promise<void>;
+export declare function infiniteScrollToBottom(page: Page, opts?: {
+    maxScrolls?: number;
+    scrollDelay?: number;
+    stabilizeMs?: number;
+}): Promise<{
+    scrollCount: number;
+    heightReached: number;
+}>;
+export declare function extractIframeContent(page: Page): Promise<Array<{
+    src: string;
+    content: string;
+    markdown: string;
+}>>;
+export declare function pierceShadowDom(page: Page): Promise<string>;