@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/shadow-dom-crawler.js

@@ -0,0 +1,132 @@
+async function enumerateShadowHosts(page) {
+  const raw = await page.evaluate(() => {
+    if (typeof globalThis.__name === "undefined") globalThis.__name = (fn) => fn;
+    function collectHosts(root, depth) {
+      if (depth > 8) return [];
+      const hosts = [];
+      const els = root.querySelectorAll("*");
+      for (const el of Array.from(els)) {
+        const htmlEl = el;
+        if (htmlEl.shadowRoot) {
+          const interactive = htmlEl.shadowRoot.querySelectorAll(
+            'button,input,select,textarea,a[href],[role="button"],[role="link"],[role="checkbox"],[role="menuitem"],[role="tab"],[role="switch"]'
+          );
+          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll("slot")).map(
+            (s) => s.name || "default"
+          );
+          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll("*")).filter(
+            (c) => c.shadowRoot
+          ).length;
+          hosts.push({
+            tagName: htmlEl.tagName.toLowerCase(),
+            hostId: htmlEl.id || htmlEl.className?.toString()?.split(" ")[0] || "",
+            shadowMode: "open",
+            depth,
+            interactiveCount: interactive.length,
+            interactiveDetails: Array.from(interactive).slice(0, 8).map((i) => {
+              const iEl = i;
+              return {
+                role: iEl.getAttribute("role") || iEl.tagName.toLowerCase(),
+                label: (iEl.getAttribute("aria-label") || iEl.getAttribute("placeholder") || iEl.textContent?.trim() || "").slice(0, 40),
+                type: iEl.getAttribute("type") || "",
+                tagName: iEl.tagName.toLowerCase()
+              };
+            }),
+            slotNames: slots,
+            childShadowHosts: nestedHosts
+          });
+          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);
+          hosts.push(...nested);
+        } else if (el.tagName.includes("-") && !el.shadowRoot) {
+          hosts.push({
+            tagName: el.tagName.toLowerCase(),
+            hostId: el.id || "",
+            shadowMode: "closed",
+            depth,
+            interactiveCount: 0,
+            interactiveDetails: [],
+            slotNames: [],
+            childShadowHosts: 0
+          });
+        }
+      }
+      return hosts;
+    }
+    return collectHosts(document, 0);
+  });
+  return raw.slice(0, 100).map((h) => ({
+    tagName: h.tagName,
+    hostId: h.hostId,
+    shadowMode: h.shadowMode,
+    depth: h.depth,
+    interactiveCount: h.interactiveCount,
+    interactiveElements: h.interactiveDetails,
+    slotNames: h.slotNames,
+    childShadowHosts: h.childShadowHosts
+  }));
+}
+async function detectWebComponentLibrary(page) {
+  return page.evaluate(() => {
+    const checks = [
+      ["sl-button", "Shoelace"],
+      ["fast-button", "Microsoft FAST"],
+      ["mwc-button", "Material Web Components"],
+      ["ion-button", "Ionic"],
+      ["vaadin-button", "Vaadin"],
+      ["fluent-button", "Fluent UI Web Components"],
+      ["hy-icon", "Hydra"]
+    ];
+    const detected = [];
+    for (const [tag, name] of checks) {
+      if (customElements.get(tag)) detected.push(name);
+    }
+    const allTags = Array.from(document.querySelectorAll("*")).map((e) => e.tagName.toLowerCase()).filter((t) => t.includes("-"));
+    const uniqueTags = Array.from(new Set(allTags));
+    if (uniqueTags.some((t) => t.startsWith("sl-")) && !detected.includes("Shoelace"))
+      detected.push("Shoelace");
+    if (uniqueTags.some((t) => t.startsWith("fast-")) && !detected.includes("Microsoft FAST"))
+      detected.push("Microsoft FAST");
+    if (uniqueTags.some((t) => t.startsWith("ion-")) && !detected.includes("Ionic"))
+      detected.push("Ionic");
+    return detected;
+  });
+}
+async function crawlShadowDom(page) {
+  const empty = {
+    shadowHostCount: 0,
+    openShadowCount: 0,
+    closedShadowCount: 0,
+    maxNestingDepth: 0,
+    hosts: [],
+    totalInteractiveInShadow: 0,
+    detectedLibraries: [],
+    blindSpots: []
+  };
+  try {
+    const [hosts, detectedLibraries] = await Promise.all([
+      enumerateShadowHosts(page),
+      detectWebComponentLibrary(page)
+    ]);
+    const openHosts = hosts.filter((h) => h.shadowMode === "open");
+    const closedHosts = hosts.filter((h) => h.shadowMode === "closed");
+    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);
+    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);
+    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];
+    return {
+      shadowHostCount: hosts.length,
+      openShadowCount: openHosts.length,
+      closedShadowCount: closedHosts.length,
+      maxNestingDepth,
+      hosts,
+      totalInteractiveInShadow,
+      detectedLibraries,
+      blindSpots
+    };
+  } catch {
+    return empty;
+  }
+}
+export {
+  crawlShadowDom
+};
+//# sourceMappingURL=shadow-dom-crawler.js.map

package/dist/shadow-dom-crawler.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/shadow-dom-crawler.ts"],
+  "sourcesContent": ["/**\n * Shadow DOM crawler \u2014 enumerate shadow hosts and detect web component libraries.\n *\n * Walks the live DOM recursively (up to depth 8) to find all open shadow roots\n * and custom elements with closed shadow roots, then classifies the component\n * framework in use. Results feed ZeTa's test generator so it can account for\n * elements that are invisible to standard Playwright locators.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface ShadowInteractiveEl {\n  role: string;\n  label: string;\n  type: string;\n  tagName: string;\n}\n\nexport interface ShadowHostInfo {\n  tagName: string;\n  hostId: string;\n  shadowMode: 'open' | 'closed';\n  depth: number;\n  interactiveCount: number;\n  interactiveElements: ShadowInteractiveEl[];\n  slotNames: string[];\n  childShadowHosts: number;\n}\n\nexport interface ShadowDomCrawlResult {\n  shadowHostCount: number;\n  openShadowCount: number;\n  closedShadowCount: number;\n  maxNestingDepth: number;\n  hosts: ShadowHostInfo[];\n  totalInteractiveInShadow: number;\n  detectedLibraries: string[];\n  blindSpots: string[];\n}\n\ninterface RawShadowHost {\n  tagName: string;\n  hostId: string;\n  shadowMode: 'open' | 'closed';\n  depth: number;\n  interactiveCount: number;\n  interactiveDetails: Array<{ role: string; label: string; type: string; tagName: string }>;\n  slotNames: string[];\n  childShadowHosts: number;\n}\n\nasync function enumerateShadowHosts(page: any): Promise<ShadowHostInfo[]> {\n  const raw: RawShadowHost[] = await page.evaluate(() => {\n    // esbuild injects __name() for named function declarations; polyfill prevents ReferenceError in browser.\n    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;\n    function collectHosts(root: Document | ShadowRoot, depth: number): RawShadowHost[] {\n      if (depth > 8) return [];\n      const hosts: RawShadowHost[] = [];\n      const els = root.querySelectorAll('*');\n      for (const el of Array.from(els)) {\n        const htmlEl = el as HTMLElement & { shadowRoot: ShadowRoot | null };\n        if (htmlEl.shadowRoot) {\n          const interactive = htmlEl.shadowRoot.querySelectorAll(\n            'button,input,select,textarea,a[href],[role=\"button\"],[role=\"link\"],[role=\"checkbox\"],[role=\"menuitem\"],[role=\"tab\"],[role=\"switch\"]'\n          );\n          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll('slot')).map(\n            (s) => (s as HTMLSlotElement).name || 'default'\n          );\n          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll('*')).filter(\n            (c) => (c as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot\n          ).length;\n          hosts.push({\n            tagName: htmlEl.tagName.toLowerCase(),\n            hostId:\n              htmlEl.id ||\n              (htmlEl.className?.toString()?.split(' ')[0]) ||\n              '',\n            shadowMode: 'open',\n            depth,\n            interactiveCount: interactive.length,\n            interactiveDetails: Array.from(interactive)\n              .slice(0, 8)\n              .map((i) => {\n                const iEl = i as HTMLElement;\n                return {\n                  role: iEl.getAttribute('role') || iEl.tagName.toLowerCase(),\n                  label: (\n                    iEl.getAttribute('aria-label') ||\n                    iEl.getAttribute('placeholder') ||\n                    iEl.textContent?.trim() ||\n                    ''\n                  ).slice(0, 40),\n                  type: iEl.getAttribute('type') || '',\n                  tagName: iEl.tagName.toLowerCase(),\n                };\n              }),\n            slotNames: slots,\n            childShadowHosts: nestedHosts,\n          });\n          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);\n          hosts.push(...nested);\n        } else if (el.tagName.includes('-') && !(el as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot) {\n          hosts.push({\n            tagName: el.tagName.toLowerCase(),\n            hostId: (el as HTMLElement).id || '',\n            shadowMode: 'closed',\n            depth,\n            interactiveCount: 0,\n            interactiveDetails: [],\n            slotNames: [],\n            childShadowHosts: 0,\n          });\n        }\n      }\n      return hosts;\n    }\n    return collectHosts(document, 0);\n  });\n\n  return raw.slice(0, 100).map((h) => ({\n    tagName: h.tagName,\n    hostId: h.hostId,\n    shadowMode: h.shadowMode,\n    depth: h.depth,\n    interactiveCount: h.interactiveCount,\n    interactiveElements: h.interactiveDetails,\n    slotNames: h.slotNames,\n    childShadowHosts: h.childShadowHosts,\n  }));\n}\n\nasync function detectWebComponentLibrary(page: any): Promise<string[]> {\n  return page.evaluate(() => {\n    const checks: [string, string][] = [\n      ['sl-button', 'Shoelace'],\n      ['fast-button', 'Microsoft FAST'],\n      ['mwc-button', 'Material Web Components'],\n      ['ion-button', 'Ionic'],\n      ['vaadin-button', 'Vaadin'],\n      ['fluent-button', 'Fluent UI Web Components'],\n      ['hy-icon', 'Hydra'],\n    ];\n    const detected: string[] = [];\n    for (const [tag, name] of checks) {\n      if (customElements.get(tag)) detected.push(name);\n    }\n    const allTags = Array.from(document.querySelectorAll('*'))\n      .map((e) => e.tagName.toLowerCase())\n      .filter((t) => t.includes('-'));\n    const uniqueTags = Array.from(new Set(allTags));\n    if (uniqueTags.some((t) => t.startsWith('sl-')) && !detected.includes('Shoelace'))\n      detected.push('Shoelace');\n    if (uniqueTags.some((t) => t.startsWith('fast-')) && !detected.includes('Microsoft FAST'))\n      detected.push('Microsoft FAST');\n    if (uniqueTags.some((t) => t.startsWith('ion-')) && !detected.includes('Ionic'))\n      detected.push('Ionic');\n    return detected;\n  });\n}\n\nexport async function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult> {\n  const empty: ShadowDomCrawlResult = {\n    shadowHostCount: 0,\n    openShadowCount: 0,\n    closedShadowCount: 0,\n    maxNestingDepth: 0,\n    hosts: [],\n    totalInteractiveInShadow: 0,\n    detectedLibraries: [],\n    blindSpots: [],\n  };\n\n  try {\n    const [hosts, detectedLibraries] = await Promise.all([\n      enumerateShadowHosts(page),\n      detectWebComponentLibrary(page),\n    ]);\n\n    const openHosts = hosts.filter((h) => h.shadowMode === 'open');\n    const closedHosts = hosts.filter((h) => h.shadowMode === 'closed');\n    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);\n    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);\n    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];\n\n    return {\n      shadowHostCount: hosts.length,\n      openShadowCount: openHosts.length,\n      closedShadowCount: closedHosts.length,\n      maxNestingDepth,\n      hosts,\n      totalInteractiveInShadow,\n      detectedLibraries,\n      blindSpots,\n    };\n  } catch {\n    return empty;\n  }\n}\n"],
+  "mappings": "AAmDA,eAAe,qBAAqB,MAAsC;AACxE,QAAM,MAAuB,MAAM,KAAK,SAAS,MAAM;AAErD,QAAI,OAAQ,WAAmB,WAAW,YAAa,CAAC,WAAmB,SAAS,CAAC,OAAY;AACjG,aAAS,aAAa,MAA6B,OAAgC;AACjF,UAAI,QAAQ,EAAG,QAAO,CAAC;AACvB,YAAM,QAAyB,CAAC;AAChC,YAAM,MAAM,KAAK,iBAAiB,GAAG;AACrC,iBAAW,MAAM,MAAM,KAAK,GAAG,GAAG;AAChC,cAAM,SAAS;AACf,YAAI,OAAO,YAAY;AACrB,gBAAM,cAAc,OAAO,WAAW;AAAA,YACpC;AAAA,UACF;AACA,gBAAM,QAAQ,MAAM,KAAK,OAAO,WAAW,iBAAiB,MAAM,CAAC,EAAE;AAAA,YACnE,CAAC,MAAO,EAAsB,QAAQ;AAAA,UACxC;AACA,gBAAM,cAAc,MAAM,KAAK,OAAO,WAAW,iBAAiB,GAAG,CAAC,EAAE;AAAA,YACtE,CAAC,MAAO,EAAsD;AAAA,UAChE,EAAE;AACF,gBAAM,KAAK;AAAA,YACT,SAAS,OAAO,QAAQ,YAAY;AAAA,YACpC,QACE,OAAO,MACN,OAAO,WAAW,SAAS,GAAG,MAAM,GAAG,EAAE,CAAC,KAC3C;AAAA,YACF,YAAY;AAAA,YACZ;AAAA,YACA,kBAAkB,YAAY;AAAA,YAC9B,oBAAoB,MAAM,KAAK,WAAW,EACvC,MAAM,GAAG,CAAC,EACV,IAAI,CAAC,MAAM;AACV,oBAAM,MAAM;AACZ,qBAAO;AAAA,gBACL,MAAM,IAAI,aAAa,MAAM,KAAK,IAAI,QAAQ,YAAY;AAAA,gBAC1D,QACE,IAAI,aAAa,YAAY,KAC7B,IAAI,aAAa,aAAa,KAC9B,IAAI,aAAa,KAAK,KACtB,IACA,MAAM,GAAG,EAAE;AAAA,gBACb,MAAM,IAAI,aAAa,MAAM,KAAK;AAAA,gBAClC,SAAS,IAAI,QAAQ,YAAY;AAAA,cACnC;AAAA,YACF,CAAC;AAAA,YACH,WAAW;AAAA,YACX,kBAAkB;AAAA,UACpB,CAAC;AACD,gBAAM,SAAS,aAAa,OAAO,YAAY,QAAQ,CAAC;AACxD,gBAAM,KAAK,GAAG,MAAM;AAAA,QACtB,WAAW,GAAG,QAAQ,SAAS,GAAG,KAAK,CAAE,GAAuD,YAAY;AAC1G,gBAAM,KAAK;AAAA,YACT,SAAS,GAAG,QAAQ,YAAY;AAAA,YAChC,QAAS,GAAmB,MAAM;AAAA,YAClC,YAAY;AAAA,YACZ;AAAA,YACA,kBAAkB;AAAA,YAClB,oBAAoB,CAAC;AAAA,YACrB,WAAW,CAAC;AAAA,YACZ,kBAAkB;AAAA,UACpB,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AACA,WAAO,aAAa,UAAU,CAAC;AAAA,EACjC,CAAC;AAED,SAAO,IAAI,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,OAAO;AAAA,IACnC,SAAS,EAAE;AAAA,IACX,QAAQ,EAAE;AAAA,IACV,YAAY,EAAE;AAAA,IACd,OAAO,EAAE;AAAA,IACT,kBAAkB,EAAE;AAAA,IACpB,qBAAqB,EAAE;AAAA,IACvB,WAAW,EAAE;AAAA,IACb,kBAAkB,EAAE;AAAA,EACtB,EAAE;AACJ;AAEA,eAAe,0BAA0B,MAA8B;AACrE,SAAO,KAAK,SAAS,MAAM;AACzB,UAAM,SAA6B;AAAA,MACjC,CAAC,aAAa,UAAU;AAAA,MACxB,CAAC,eAAe,gBAAgB;AAAA,MAChC,CAAC,cAAc,yBAAyB;AAAA,MACxC,CAAC,cAAc,OAAO;AAAA,MACtB,CAAC,iBAAiB,QAAQ;AAAA,MAC1B,CAAC,iBAAiB,0BAA0B;AAAA,MAC5C,CAAC,WAAW,OAAO;AAAA,IACrB;AACA,UAAM,WAAqB,CAAC;AAC5B,eAAW,CAAC,KAAK,IAAI,KAAK,QAAQ;AAChC,UAAI,eAAe,IAAI,GAAG,EAAG,UAAS,KAAK,IAAI;AAAA,IACjD;AACA,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,GAAG,CAAC,EACtD,IAAI,CAAC,MAAM,EAAE,QAAQ,YAAY,CAAC,EAClC,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,CAAC;AAChC,UAAM,aAAa,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC;AAC9C,QAAI,WAAW,KAAK,CAAC,MAAM,EAAE,WAAW,KAAK,CAAC,KAAK,CAAC,SAAS,SAAS,UAAU;AAC9E,eAAS,KAAK,UAAU;AAC1B,QAAI,WAAW,KAAK,CAAC,MAAM,EAAE,WAAW,OAAO,CAAC,KAAK,CAAC,SAAS,SAAS,gBAAgB;AACtF,eAAS,KAAK,gBAAgB;AAChC,QAAI,WAAW,KAAK,CAAC,MAAM,EAAE,WAAW,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,OAAO;AAC5E,eAAS,KAAK,OAAO;AACvB,WAAO;AAAA,EACT,CAAC;AACH;AAEA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,QAA8B;AAAA,IAClC,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,mBAAmB;AAAA,IACnB,iBAAiB;AAAA,IACjB,OAAO,CAAC;AAAA,IACR,0BAA0B;AAAA,IAC1B,mBAAmB,CAAC;AAAA,IACpB,YAAY,CAAC;AAAA,EACf;AAEA,MAAI;AACF,UAAM,CAAC,OAAO,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,MACnD,qBAAqB,IAAI;AAAA,MACzB,0BAA0B,IAAI;AAAA,IAChC,CAAC;AAED,UAAM,YAAY,MAAM,OAAO,CAAC,MAAM,EAAE,eAAe,MAAM;AAC7D,UAAM,cAAc,MAAM,OAAO,CAAC,MAAM,EAAE,eAAe,QAAQ;AACjE,UAAM,kBAAkB,MAAM,OAAO,CAAC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,KAAK,GAAG,CAAC;AAC1E,UAAM,2BAA2B,UAAU,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,kBAAkB,CAAC;AACzF,UAAM,aAAa,CAAC,GAAG,IAAI,IAAI,YAAY,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjE,WAAO;AAAA,MACL,iBAAiB,MAAM;AAAA,MACvB,iBAAiB,UAAU;AAAA,MAC3B,mBAAmB,YAAY;AAAA,MAC/B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/sitemap-writer.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Sitemap writer — generate XML sitemap from crawled screens.
+ *
+ * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).
+ * Splits into 50k-URL files per the spec when needed.
+ * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.
+ */
+export interface SitemapUrl {
+    loc: string;
+    lastmod?: string;
+    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+    priority?: number;
+}
+export declare function writeSitemap(urls: SitemapUrl[], outputPath: string): Promise<void>;
+/** Convert crawled screen records to SitemapUrl objects */
+export declare function screensToSitemapUrls(screens: Array<{
+    url: string;
+    updatedAt?: Date;
+    depth?: number;
+}>): SitemapUrl[];
+export declare function buildSitemapString(urls: SitemapUrl[]): string;

package/dist/sitemap-writer.js

@@ -0,0 +1,47 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+const XML_HEADER = '<?xml version="1.0" encoding="UTF-8"?>\n';
+const SITEMAP_NS = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
+function escape(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function buildSitemapXml(urls) {
+  const entries = urls.map((u) => [
+    "  <url>",
+    `    <loc>${escape(u.loc)}</loc>`,
+    u.lastmod ? `    <lastmod>${u.lastmod}</lastmod>` : "",
+    u.changefreq ? `    <changefreq>${u.changefreq}</changefreq>` : "",
+    u.priority !== void 0 ? `    <priority>${u.priority.toFixed(1)}</priority>` : "",
+    "  </url>"
+  ].filter(Boolean).join("\n")).join("\n");
+  return `${XML_HEADER}<urlset ${SITEMAP_NS}>
+${entries}
+</urlset>`;
+}
+async function writeSitemap(urls, outputPath) {
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, buildSitemapXml(urls), "utf8");
+}
+function screensToSitemapUrls(screens) {
+  return screens.filter((s) => {
+    try {
+      return new URL(s.url).protocol.startsWith("http");
+    } catch {
+      return false;
+    }
+  }).map((s) => ({
+    loc: s.url,
+    lastmod: s.updatedAt ? s.updatedAt.toISOString().split("T")[0] : void 0,
+    changefreq: "weekly",
+    priority: s.depth === 0 ? 1 : s.depth === 1 ? 0.8 : 0.6
+  }));
+}
+function buildSitemapString(urls) {
+  return buildSitemapXml(urls);
+}
+export {
+  buildSitemapString,
+  screensToSitemapUrls,
+  writeSitemap
+};
+//# sourceMappingURL=sitemap-writer.js.map

package/dist/sitemap-writer.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/sitemap-writer.ts"],
+  "sourcesContent": ["/**\n * Sitemap writer \u2014 generate XML sitemap from crawled screens.\n *\n * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).\n * Splits into 50k-URL files per the spec when needed.\n * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.\n */\n\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\n\nexport interface SitemapUrl {\n  loc: string;\n  lastmod?: string;\n  changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';\n  priority?: number; // 0.0\u20131.0\n}\n\nconst XML_HEADER = '<?xml version=\"1.0\" encoding=\"UTF-8\"?>\\n';\nconst SITEMAP_NS = 'xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"';\n\nfunction escape(s: string): string {\n  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/\"/g, '&quot;').replace(/'/g, '&apos;');\n}\n\nfunction buildSitemapXml(urls: SitemapUrl[]): string {\n  const entries = urls.map(u => [\n    '  <url>',\n    `    <loc>${escape(u.loc)}</loc>`,\n    u.lastmod ? `    <lastmod>${u.lastmod}</lastmod>` : '',\n    u.changefreq ? `    <changefreq>${u.changefreq}</changefreq>` : '',\n    u.priority !== undefined ? `    <priority>${u.priority.toFixed(1)}</priority>` : '',\n    '  </url>',\n  ].filter(Boolean).join('\\n')).join('\\n');\n\n  return `${XML_HEADER}<urlset ${SITEMAP_NS}>\\n${entries}\\n</urlset>`;\n}\n\nexport async function writeSitemap(\n  urls: SitemapUrl[],\n  outputPath: string,\n): Promise<void> {\n  await fs.mkdir(path.dirname(outputPath), { recursive: true });\n  await fs.writeFile(outputPath, buildSitemapXml(urls), 'utf8');\n}\n\n/** Convert crawled screen records to SitemapUrl objects */\nexport function screensToSitemapUrls(\n  screens: Array<{ url: string; updatedAt?: Date; depth?: number }>,\n): SitemapUrl[] {\n  return screens\n    .filter(s => {\n      try { return new URL(s.url).protocol.startsWith('http'); } catch { return false; }\n    })\n    .map(s => ({\n      loc: s.url,\n      lastmod: s.updatedAt ? s.updatedAt.toISOString().split('T')[0] : undefined,\n      changefreq: 'weekly' as const,\n      priority: s.depth === 0 ? 1.0 : s.depth === 1 ? 0.8 : 0.6,\n    }));\n}\n\nexport function buildSitemapString(urls: SitemapUrl[]): string {\n  return buildSitemapXml(urls);\n}\n"],
+  "mappings": "AAQA,YAAY,QAAQ;AACpB,YAAY,UAAU;AAStB,MAAM,aAAa;AACnB,MAAM,aAAa;AAEnB,SAAS,OAAO,GAAmB;AACjC,SAAO,EAAE,QAAQ,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,QAAQ,MAAM,MAAM,EAAE,QAAQ,MAAM,QAAQ,EAAE,QAAQ,MAAM,QAAQ;AAC5H;AAEA,SAAS,gBAAgB,MAA4B;AACnD,QAAM,UAAU,KAAK,IAAI,OAAK;AAAA,IAC5B;AAAA,IACA,YAAY,OAAO,EAAE,GAAG,CAAC;AAAA,IACzB,EAAE,UAAU,gBAAgB,EAAE,OAAO,eAAe;AAAA,IACpD,EAAE,aAAa,mBAAmB,EAAE,UAAU,kBAAkB;AAAA,IAChE,EAAE,aAAa,SAAY,iBAAiB,EAAE,SAAS,QAAQ,CAAC,CAAC,gBAAgB;AAAA,IACjF;AAAA,EACF,EAAE,OAAO,OAAO,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI;AAEvC,SAAO,GAAG,UAAU,WAAW,UAAU;AAAA,EAAM,OAAO;AAAA;AACxD;AAEA,eAAsB,aACpB,MACA,YACe;AACf,QAAM,GAAG,MAAM,KAAK,QAAQ,UAAU,GAAG,EAAE,WAAW,KAAK,CAAC;AAC5D,QAAM,GAAG,UAAU,YAAY,gBAAgB,IAAI,GAAG,MAAM;AAC9D;AAGO,SAAS,qBACd,SACc;AACd,SAAO,QACJ,OAAO,OAAK;AACX,QAAI;AAAE,aAAO,IAAI,IAAI,EAAE,GAAG,EAAE,SAAS,WAAW,MAAM;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EACnF,CAAC,EACA,IAAI,QAAM;AAAA,IACT,KAAK,EAAE;AAAA,IACP,SAAS,EAAE,YAAY,EAAE,UAAU,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,IAAI;AAAA,IACjE,YAAY;AAAA,IACZ,UAAU,EAAE,UAAU,IAAI,IAAM,EAAE,UAAU,IAAI,MAAM;AAAA,EACxD,EAAE;AACN;AAEO,SAAS,mBAAmB,MAA4B;AAC7D,SAAO,gBAAgB,IAAI;AAC7B;",
+  "names": []
+}

package/dist/sitemap.d.ts

@@ -0,0 +1,51 @@
+import { ProxyAgent } from 'undici';
+/**
+ * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
+ * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
+ */
+export declare function sameSite(candidateUrl: string, siteOrigin: string): boolean;
+/**
+ * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
+ * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
+ * and deduped. Pure function — no I/O — for direct unit testing without a
+ * real HTTP fetch.
+ */
+export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls?: number): {
+    pageUrls: string[];
+    subSitemapUrls: string[];
+};
+/**
+ * Fetches and parses a sitemap (or sitemap index, recursed up to a small
+ * depth) into a flat, deduped list of same-origin URLs.
+ */
+export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any, maxUrls?: number): Promise<string[]>;
+/**
+ * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
+ * explicit override) and returns the URLs it lists. Never throws.
+ */
+export declare function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]>;
+/**
+ * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
+ * DataImpulse-style usernames contain commas (country codes) that break new URL().
+ * Returns undefined for no/auto proxy — the caller then fetches directly.
+ */
+export declare function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined;
+/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
+export declare function parseRobotsSitemaps(robotsTxt: string, origin: string): string[];
+export interface UrlCandidate {
+    url: string;
+    source: 'sitemap' | 'robots' | 'link';
+}
+/**
+ * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
+ * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
+ * candidate list the user can review and seed before launching the Playwright crawl.
+ * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
+ */
+export declare function discoverProjectUrls(appUrl: string, opts?: {
+    sitemapUrl?: string;
+    proxyUrl?: string;
+}): Promise<{
+    candidates: UrlCandidate[];
+    sitemapFound: boolean;
+}>;

package/dist/sitemap.js

@@ -0,0 +1,160 @@
+import { XMLParser } from "fast-xml-parser";
+import { ProxyAgent } from "undici";
+import { assertCrawlUrlSafe } from "@taodigital/core/dist/ssrf-guard.js";
+import { logger } from "./logger.js";
+const FETCH_TIMEOUT_MS = 1e4;
+const MAX_SITEMAP_INDEX_DEPTH = 3;
+const MAX_SUB_SITEMAPS = 50;
+async function getSitemapMaxUrls() {
+  const { getSystemConfig } = await import("@taodigital/core");
+  const cfg = await getSystemConfig("crawl_tuning");
+  return parseInt(String(cfg?.sitemapMaxUrls ?? "5000"), 10);
+}
+function toArray(x) {
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
+}
+function hostKey(hostname) {
+  return hostname.replace(/^www\./i, "").toLowerCase();
+}
+function sameSite(candidateUrl, siteOrigin) {
+  try {
+    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
+    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
+  } catch {
+    return false;
+  }
+}
+function extractUrlsFromParsedSitemap(parsed, origin, maxUrls = 5e3) {
+  const rawPageUrls = [];
+  let subSitemapUrls = [];
+  if (parsed?.sitemapindex) {
+    subSitemapUrls = toArray(parsed.sitemapindex.sitemap).map((s) => s?.loc).filter((loc) => typeof loc === "string").slice(0, MAX_SUB_SITEMAPS);
+  }
+  if (parsed?.urlset) {
+    rawPageUrls.push(
+      ...toArray(parsed.urlset.url).map((u) => u?.loc).filter((loc) => typeof loc === "string")
+    );
+  }
+  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };
+}
+async function fetchAndParseXml(url, dispatcher) {
+  try {
+    assertCrawlUrlSafe(url);
+  } catch {
+    return null;
+  }
+  try {
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+sitemap-discovery)" },
+      ...dispatcher ? { dispatcher } : {}
+    });
+    if (!res.ok) return null;
+    const text = await res.text();
+    return new XMLParser().parse(text);
+  } catch {
+    return null;
+  }
+}
+async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher, maxUrls) {
+  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  if (maxUrls === void 0) maxUrls = await getSitemapMaxUrls();
+  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
+  if (!parsed) return [];
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);
+  const collected = [...pageUrls];
+  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))
+    );
+    for (const sub of subResults) collected.push(...sub);
+  }
+  return [...new Set(collected)].slice(0, maxUrls);
+}
+async function discoverSitemapUrls(appUrl, sitemapUrl) {
+  let origin;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return [];
+  }
+  const target = sitemapUrl || `${origin}/sitemap.xml`;
+  try {
+    return await fetchSitemapUrls(target, origin);
+  } catch (err) {
+    logger.warn({ err: String(err), target }, "[sitemap] discovery failed, continuing without it");
+    return [];
+  }
+}
+function buildProxyDispatcher(proxyUrl) {
+  if (!proxyUrl || proxyUrl === "auto") return void 0;
+  try {
+    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (!m) return new ProxyAgent(proxyUrl);
+    const [, scheme, user, pass, hostPort] = m;
+    const uri = `${scheme}://${hostPort}`;
+    if (user) {
+      const token = "Basic " + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? "")}`).toString("base64");
+      return new ProxyAgent({ uri, token });
+    }
+    return new ProxyAgent({ uri });
+  } catch {
+    return void 0;
+  }
+}
+function parseRobotsSitemaps(robotsTxt, origin) {
+  const out = [];
+  for (const line of robotsTxt.split(/\r?\n/)) {
+    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
+    if (m && sameSite(m[1], origin)) out.push(m[1]);
+  }
+  return [...new Set(out)];
+}
+async function discoverProjectUrls(appUrl, opts) {
+  let origin;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return { candidates: [], sitemapFound: false };
+  }
+  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  const maxUrls = await getSitemapMaxUrls();
+  let robotsSitemaps = [];
+  try {
+    assertCrawlUrlSafe(`${origin}/robots.txt`);
+    const res = await fetch(`${origin}/robots.txt`, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+url-discovery)" },
+      ...dispatcher ? { dispatcher } : {}
+    });
+    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
+  } catch {
+  }
+  const robotsSet = new Set(robotsSitemaps);
+  const sitemapTargets = [.../* @__PURE__ */ new Set([
+    ...opts?.sitemapUrl ? [opts.sitemapUrl] : [],
+    ...robotsSitemaps,
+    `${origin}/sitemap.xml`
+  ])];
+  const seen = /* @__PURE__ */ new Map();
+  let sitemapFound = false;
+  for (const sm of sitemapTargets) {
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);
+    if (urls.length) sitemapFound = true;
+    const source = robotsSet.has(sm) ? "robots" : "sitemap";
+    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
+  }
+  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };
+}
+export {
+  buildProxyDispatcher,
+  discoverProjectUrls,
+  discoverSitemapUrls,
+  extractUrlsFromParsedSitemap,
+  fetchSitemapUrls,
+  parseRobotsSitemaps,
+  sameSite
+};
+//# sourceMappingURL=sitemap.js.map

package/dist/sitemap.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/sitemap.ts"],
+  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core/dist/ssrf-guard.js';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\n\nasync function getSitemapMaxUrls(): Promise<number> {\n  const { getSystemConfig } = await import('@detiq/core');\n  const cfg = await getSystemConfig('crawl_tuning');\n  return parseInt(String(cfg?.sitemapMaxUrls ?? '5000'), 10);\n}\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls = 5000): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any, maxUrls?: number): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  if (maxUrls === undefined) maxUrls = await getSitemapMaxUrls();\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);\n  const collected = [...pageUrls];\n\n  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {\n    const subResults = await Promise.all(\n      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))\n    );\n    for (const sub of subResults) collected.push(...sub);\n  }\n\n  return [...new Set(collected)].slice(0, maxUrls);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n  const maxUrls = await getSitemapMaxUrls();\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };\n}\n"],
+  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AAEzB,eAAe,oBAAqC;AAClD,QAAM,EAAE,gBAAgB,IAAI,MAAM,OAAO,aAAa;AACtD,QAAM,MAAM,MAAM,gBAAgB,cAAc;AAChD,SAAO,SAAS,OAAO,KAAK,kBAAkB,MAAM,GAAG,EAAE;AAC3D;AAEA,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAgB,UAAU,KAAwD;AAC1I,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,eAAe;AAChF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAkB,SAAqC;AAC3I,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,MAAI,YAAY,OAAW,WAAU,MAAM,kBAAkB;AAC7D,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,QAAQ,OAAO;AACzF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,MAAI,eAAe,SAAS,KAAK,UAAU,SAAS,SAAS;AAC3D,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,eAAe,MAAM,GAAG,EAAE,EAAE,IAAI,YAAU,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,YAAY,OAAO,CAAC;AAAA,IAC5G;AACA,eAAW,OAAO,WAAY,WAAU,KAAK,GAAG,GAAG;AAAA,EACrD;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO;AACjD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AACtD,QAAM,UAAU,MAAM,kBAAkB;AAGxC,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,YAAY,OAAO;AACtE,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,aAAa;AAC1E;",
+  "names": []
+}

package/dist/skyvern.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Skyvern Vision Fallback
+ *
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
+ * government portals. It's the "break glass" option when Stagehand fails.
+ *
+ * Production setup (Phase 4+):
+ *   1. Self-host: https://github.com/Skyvern-AI/skyvern
+ *   2. Or use cloud: https://app.skyvern.com
+ *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
+ *
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
+ * Use only when the fast path fails.
+ */
+export interface SkyvernTask {
+    taskDescription: string;
+    url: string;
+    credentials?: {
+        username: string;
+        password: string;
+        oneTimeCode?: string;
+    };
+}
+export interface SkyvernConfig {
+    apiUrl: string;
+    apiKey: string;
+    llmKey?: string;
+    llmApiKey?: string;
+}
+export interface SkyvernStep {
+    order: number;
+    action_type: string;
+    element_description?: string;
+    reasoning?: string;
+    success: boolean;
+    screenshot_url?: string;
+    duration_ms?: number;
+    error?: string;
+}
+export interface SkyvernNavigateResult {
+    screenshotUrls: string[];
+    visitedUrls: string[];
+    status: 'success' | 'failed' | 'partial';
+    steps?: SkyvernStep[];
+    taskId?: string;
+    durationMs?: number;
+    triggerReason?: string;
+}
+/**
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
+ */
+export declare function skyvernNavigate(startUrl: string, navigationGoals: string[], tenantConfig?: SkyvernConfig, triggerReason?: string): Promise<SkyvernNavigateResult>;
+export declare function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'>;