@detiq/crawler 1.1.13 → 1.1.15

package/dist/rate-limiter.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/rate-limiter.ts"],
-  "sourcesContent": ["/**\n * Domain-aware rate limiter \u2014 ensures the crawler respects per-domain rate limits.\n *\n * Uses a token bucket algorithm per domain. When the bucket is empty, requests\n * wait until a token is available. Tokens refill at a configurable rate.\n *\n * Default: 2 requests/second per domain, burst of 5.\n * Can be configured per-domain via the rateMap (domain \u2192 requests/second).\n */\n\nexport interface RateLimiterOpts {\n  defaultRps?: number;        // requests per second (default 2)\n  burstSize?: number;         // max burst (default 5)\n  rateMap?: Record<string, number>; // domain-specific overrides\n}\n\ninterface Bucket {\n  tokens: number;\n  lastRefill: number; // Date.now()\n}\n\nfunction extractDomain(url: string): string {\n  try { return new URL(url).hostname; } catch { return url; }\n}\n\nexport class DomainRateLimiter {\n  private buckets = new Map<string, Bucket>();\n  private readonly defaultRps: number;\n  private readonly burstSize: number;\n  private readonly rateMap: Record<string, number>;\n\n  constructor(opts: RateLimiterOpts = {}) {\n    this.defaultRps = opts.defaultRps ?? 2;\n    this.burstSize = opts.burstSize ?? 5;\n    this.rateMap = opts.rateMap ?? {};\n  }\n\n  private getBucket(domain: string): Bucket {\n    if (!this.buckets.has(domain)) {\n      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });\n    }\n    return this.buckets.get(domain)!;\n  }\n\n  private refill(domain: string, bucket: Bucket): void {\n    const rps = this.rateMap[domain] ?? this.defaultRps;\n    const now = Date.now();\n    const elapsed = (now - bucket.lastRefill) / 1000;\n    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);\n    bucket.lastRefill = now;\n  }\n\n  /** Wait until a token is available for the given URL's domain. */\n  async throttle(url: string): Promise<void> {\n    const domain = extractDomain(url);\n    const bucket = this.getBucket(domain);\n    this.refill(domain, bucket);\n\n    if (bucket.tokens >= 1) {\n      bucket.tokens -= 1;\n      return;\n    }\n\n    // Calculate wait time until next token\n    const rps = this.rateMap[domain] ?? this.defaultRps;\n    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);\n    await new Promise(r => setTimeout(r, waitMs));\n    bucket.tokens = 0;\n    bucket.lastRefill = Date.now();\n  }\n\n  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */\n  setRate(domain: string, rps: number): void {\n    this.rateMap[domain] = rps;\n    const bucket = this.getBucket(domain);\n    bucket.tokens = 0; // drain immediately on rate change\n  }\n\n  stats(): Record<string, { tokens: number; rps: number }> {\n    const result: Record<string, any> = {};\n    for (const [domain, bucket] of this.buckets) {\n      this.refill(domain, bucket);\n      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };\n    }\n    return result;\n  }\n}\n"],
-  "mappings": "AAqBA,SAAS,cAAc,KAAqB;AAC1C,MAAI;AAAE,WAAO,IAAI,IAAI,GAAG,EAAE;AAAA,EAAU,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC5D;AAEO,MAAM,kBAAkB;AAAA,EACrB,UAAU,oBAAI,IAAoB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,OAAwB,CAAC,GAAG;AACtC,SAAK,aAAa,KAAK,cAAc;AACrC,SAAK,YAAY,KAAK,aAAa;AACnC,SAAK,UAAU,KAAK,WAAW,CAAC;AAAA,EAClC;AAAA,EAEQ,UAAU,QAAwB;AACxC,QAAI,CAAC,KAAK,QAAQ,IAAI,MAAM,GAAG;AAC7B,WAAK,QAAQ,IAAI,QAAQ,EAAE,QAAQ,KAAK,WAAW,YAAY,KAAK,IAAI,EAAE,CAAC;AAAA,IAC7E;AACA,WAAO,KAAK,QAAQ,IAAI,MAAM;AAAA,EAChC;AAAA,EAEQ,OAAO,QAAgB,QAAsB;AACnD,UAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,KAAK;AACzC,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,WAAW,MAAM,OAAO,cAAc;AAC5C,WAAO,SAAS,KAAK,IAAI,KAAK,WAAW,OAAO,SAAS,UAAU,GAAG;AACtE,WAAO,aAAa;AAAA,EACtB;AAAA;AAAA,EAGA,MAAM,SAAS,KAA4B;AACzC,UAAM,SAAS,cAAc,GAAG;AAChC,UAAM,SAAS,KAAK,UAAU,MAAM;AACpC,SAAK,OAAO,QAAQ,MAAM;AAE1B,QAAI,OAAO,UAAU,GAAG;AACtB,aAAO,UAAU;AACjB;AAAA,IACF;AAGA,UAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,KAAK;AACzC,UAAM,SAAS,KAAK,MAAM,IAAI,OAAO,UAAU,MAAM,GAAI;AACzD,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,MAAM,CAAC;AAC5C,WAAO,SAAS;AAChB,WAAO,aAAa,KAAK,IAAI;AAAA,EAC/B;AAAA;AAAA,EAGA,QAAQ,QAAgB,KAAmB;AACzC,SAAK,QAAQ,MAAM,IAAI;AACvB,UAAM,SAAS,KAAK,UAAU,MAAM;AACpC,WAAO,SAAS;AAAA,EAClB;AAAA,EAEA,QAAyD;AACvD,UAAM,SAA8B,CAAC;AACrC,eAAW,CAAC,QAAQ,MAAM,KAAK,KAAK,SAAS;AAC3C,WAAK,OAAO,QAAQ,MAAM;AAC1B,aAAO,MAAM,IAAI,EAAE,QAAQ,KAAK,MAAM,OAAO,SAAS,GAAG,IAAI,KAAK,KAAK,KAAK,QAAQ,MAAM,KAAK,KAAK,WAAW;AAAA,IACjH;AACA,WAAO;AAAA,EACT;AACF;",
+  "sourcesContent": ["/**\n * Domain-aware rate limiter \u2014 ensures the crawler respects per-domain rate limits.\n *\n * Uses a token bucket algorithm per domain. When the bucket is empty, requests\n * wait until a token is available. Tokens refill at a configurable rate.\n *\n * Default: 2 requests/second per domain, burst of 5.\n * Can be configured per-domain via the rateMap (domain \u2192 requests/second).\n */\n\nexport interface RateLimiterOpts {\n  defaultRps?: number;        // requests per second (default 2)\n  burstSize?: number;         // max burst (default 5)\n  rateMap?: Record<string, number>; // domain-specific overrides\n}\n\ninterface Bucket {\n  tokens: number;\n  lastRefill: number; // Date.now()\n}\n\nfunction extractDomain(url: string): string {\n  try { return new URL(url).hostname; } catch { return url; }\n}\n\nexport class DomainRateLimiter {\n  private buckets = new Map<string, Bucket>();\n  private readonly defaultRps: number;\n  private readonly burstSize: number;\n  private readonly rateMap: Record<string, number>;\n\n  constructor(opts: RateLimiterOpts = {}) {\n    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? '') || 2);\n    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? '') || 5);\n    this.rateMap = opts.rateMap ?? {};\n  }\n\n  private getBucket(domain: string): Bucket {\n    if (!this.buckets.has(domain)) {\n      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });\n    }\n    return this.buckets.get(domain)!;\n  }\n\n  private refill(domain: string, bucket: Bucket): void {\n    const rps = this.rateMap[domain] ?? this.defaultRps;\n    const now = Date.now();\n    const elapsed = (now - bucket.lastRefill) / 1000;\n    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);\n    bucket.lastRefill = now;\n  }\n\n  /** Wait until a token is available for the given URL's domain. */\n  async throttle(url: string): Promise<void> {\n    const domain = extractDomain(url);\n    const bucket = this.getBucket(domain);\n    this.refill(domain, bucket);\n\n    if (bucket.tokens >= 1) {\n      bucket.tokens -= 1;\n      return;\n    }\n\n    // Calculate wait time until next token\n    const rps = this.rateMap[domain] ?? this.defaultRps;\n    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);\n    await new Promise(r => setTimeout(r, waitMs));\n    bucket.tokens = 0;\n    bucket.lastRefill = Date.now();\n  }\n\n  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */\n  setRate(domain: string, rps: number): void {\n    this.rateMap[domain] = rps;\n    const bucket = this.getBucket(domain);\n    bucket.tokens = 0; // drain immediately on rate change\n  }\n\n  stats(): Record<string, { tokens: number; rps: number }> {\n    const result: Record<string, any> = {};\n    for (const [domain, bucket] of this.buckets) {\n      this.refill(domain, bucket);\n      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };\n    }\n    return result;\n  }\n}\n"],
+  "mappings": "AAqBA,SAAS,cAAc,KAAqB;AAC1C,MAAI;AAAE,WAAO,IAAI,IAAI,GAAG,EAAE;AAAA,EAAU,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC5D;AAEO,MAAM,kBAAkB;AAAA,EACrB,UAAU,oBAAI,IAAoB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,OAAwB,CAAC,GAAG;AACtC,SAAK,aAAa,KAAK,eAAe,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AACtG,SAAK,YAAY,KAAK,cAAc,SAAS,QAAQ,IAAI,iCAAiC,EAAE,KAAK;AACjG,SAAK,UAAU,KAAK,WAAW,CAAC;AAAA,EAClC;AAAA,EAEQ,UAAU,QAAwB;AACxC,QAAI,CAAC,KAAK,QAAQ,IAAI,MAAM,GAAG;AAC7B,WAAK,QAAQ,IAAI,QAAQ,EAAE,QAAQ,KAAK,WAAW,YAAY,KAAK,IAAI,EAAE,CAAC;AAAA,IAC7E;AACA,WAAO,KAAK,QAAQ,IAAI,MAAM;AAAA,EAChC;AAAA,EAEQ,OAAO,QAAgB,QAAsB;AACnD,UAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,KAAK;AACzC,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,WAAW,MAAM,OAAO,cAAc;AAC5C,WAAO,SAAS,KAAK,IAAI,KAAK,WAAW,OAAO,SAAS,UAAU,GAAG;AACtE,WAAO,aAAa;AAAA,EACtB;AAAA;AAAA,EAGA,MAAM,SAAS,KAA4B;AACzC,UAAM,SAAS,cAAc,GAAG;AAChC,UAAM,SAAS,KAAK,UAAU,MAAM;AACpC,SAAK,OAAO,QAAQ,MAAM;AAE1B,QAAI,OAAO,UAAU,GAAG;AACtB,aAAO,UAAU;AACjB;AAAA,IACF;AAGA,UAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,KAAK;AACzC,UAAM,SAAS,KAAK,MAAM,IAAI,OAAO,UAAU,MAAM,GAAI;AACzD,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,MAAM,CAAC;AAC5C,WAAO,SAAS;AAChB,WAAO,aAAa,KAAK,IAAI;AAAA,EAC/B;AAAA;AAAA,EAGA,QAAQ,QAAgB,KAAmB;AACzC,SAAK,QAAQ,MAAM,IAAI;AACvB,UAAM,SAAS,KAAK,UAAU,MAAM;AACpC,WAAO,SAAS;AAAA,EAClB;AAAA,EAEA,QAAyD;AACvD,UAAM,SAA8B,CAAC;AACrC,eAAW,CAAC,QAAQ,MAAM,KAAK,KAAK,SAAS;AAC3C,WAAK,OAAO,QAAQ,MAAM;AAC1B,aAAO,MAAM,IAAI,EAAE,QAAQ,KAAK,MAAM,OAAO,SAAS,GAAG,IAAI,KAAK,KAAK,KAAK,QAAQ,MAAM,KAAK,KAAK,WAAW;AAAA,IACjH;AACA,WAAO;AAAA,EACT;AACF;",
   "names": []
 }

package/dist/screen-reader.js

@@ -1,5 +1,6 @@
 async function readScreen(page) {
   return page.evaluate(() => {
+    if (typeof globalThis.__name === "undefined") globalThis.__name = (fn) => fn;
     const out = [];
     const seen = /* @__PURE__ */ new Set();
     const noiseLabel = /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;

package/dist/screen-reader.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/screen-reader.ts"],
-  "sourcesContent": ["import type { Page } from 'playwright';\n\n/**\n * Reads a page as a structured outline (cheap + reliable), the way\n * Playwright MCP / the accessibility tree works \u2014 NOT by sending pixels to a\n * vision model. Vision is a fallback for image-only pages (added later).\n *\n * Returns the interactive elements found, which the Screen Reader Agent then\n * enriches with MEANING and saves to the Zeta Core.\n */\nexport interface RawElement {\n  role: string;\n  label: string;\n  selectorHint: string;\n  expectedData?: string;\n}\n\nexport async function readScreen(page: Page): Promise<RawElement[]> {\n  return page.evaluate(() => {\n    const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];\n    const seen = new Set<string>();\n\n    // Noise patterns to skip \u2014 cookie banners, footer chrome, version strings\n    // Intentionally excludes standalone \"close\" and \"copy\" \u2014 too many false positives\n    // (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets)\n    const noiseLabel =\n      /^(cookie|consent|privacy policy|terms of use|copyright|\u00A9|\\bv\\d+\\.\\d+|got it|dismiss|skip to|powered by|copied|\\s*$)/i;\n\n    // Include ARIA-role elements \u2014 Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags\n    const nodes = document.querySelectorAll(\n      'input:not([type=hidden]), select, textarea, button, a[href], ' +\n      '[role=\"button\"], [role=\"combobox\"], [role=\"slider\"], [role=\"switch\"], ' +\n      '[role=\"tab\"], [role=\"menuitem\"], [role=\"searchbox\"], [role=\"spinbutton\"]'\n    );\n\n    nodes.forEach((el) => {\n      const htmlEl = el as HTMLElement;\n\n      // Skip invisible elements\n      const style = window.getComputedStyle(htmlEl);\n      if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return;\n      const rect = el.getBoundingClientRect();\n      // Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links)\n      const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));\n      if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;\n\n      // Derive role\n      const tag = el.tagName.toLowerCase();\n      const role =\n        el.getAttribute('role') ||\n        (tag === 'input' ? ((el as HTMLInputElement).type || 'text') : tag);\n\n      // Derive the best human-readable label (prefer aria > placeholder > inner text)\n      const raw = (\n        el.getAttribute('aria-label') ||\n        el.getAttribute('placeholder') ||\n        (el.getAttribute('title') ?? '') ||\n        htmlEl.innerText?.replace(/\\s+/g, ' ').trim() ||\n        el.getAttribute('name') ||\n        ''\n      ).substring(0, 80);\n\n      // Skip empty, noise patterns, or very long labels (likely container text)\n      if (!raw || noiseLabel.test(raw) || raw.length > 70) return;\n\n      // Deduplicate by role+normalised label\n      const key = `${role}:${raw.toLowerCase()}`;\n      if (seen.has(key)) return;\n      seen.add(key);\n\n      let expectedData: string | undefined;\n      if (tag === 'input') {\n        const input = el as HTMLInputElement;\n        if (input.type === 'email' || /email/i.test(raw)) expectedData = 'valid_email';\n        else if (input.type === 'password' || /password/i.test(raw)) expectedData = 'password';\n        else if (input.type === 'tel' || /phone|mobile/i.test(raw)) expectedData = 'phone';\n        else if (input.type === 'date' || /\\bdate\\b/i.test(raw)) expectedData = 'date';\n        else if (/\\b(code|otp|token)\\b/i.test(raw)) expectedData = 'one_time_code';\n        else if (/\\b(search|query)\\b/i.test(raw)) expectedData = 'search_term';\n      }\n\n      out.push({ role, label: raw, selectorHint: tag, expectedData });\n    });\n\n    // Prioritise: form fields > buttons > links; within each group keep order\n    const priority = (e: typeof out[0]) => {\n      if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)) return 0;\n      if (e.role === 'button' || e.role === 'submit') return 1;\n      return 2;\n    };\n    out.sort((a, b) => priority(a) - priority(b));\n\n    return out.slice(0, 25); // cap at 25 \u2014 enough context, not overwhelming\n  });\n}\n"],
-  "mappings": "AAiBA,eAAsB,WAAW,MAAmC;AAClE,SAAO,KAAK,SAAS,MAAM;AACzB,UAAM,MAAsF,CAAC;AAC7F,UAAM,OAAO,oBAAI,IAAY;AAK7B,UAAM,aACJ;AAGF,UAAM,QAAQ,SAAS;AAAA,MACrB;AAAA,IAGF;AAEA,UAAM,QAAQ,CAAC,OAAO;AACpB,YAAM,SAAS;AAGf,YAAM,QAAQ,OAAO,iBAAiB,MAAM;AAC5C,UAAI,MAAM,YAAY,UAAU,MAAM,eAAe,YAAY,MAAM,YAAY,IAAK;AACxF,YAAM,OAAO,GAAG,sBAAsB;AAEtC,YAAM,qBAAqB,CAAC,EAAE,GAAG,aAAa,YAAY,KAAK,GAAG,aAAa,iBAAiB;AAChG,UAAI,KAAK,UAAU,KAAK,KAAK,WAAW,KAAK,CAAC,mBAAoB;AAGlE,YAAM,MAAM,GAAG,QAAQ,YAAY;AACnC,YAAM,OACJ,GAAG,aAAa,MAAM,MACrB,QAAQ,UAAY,GAAwB,QAAQ,SAAU;AAGjE,YAAM,OACJ,GAAG,aAAa,YAAY,KAC5B,GAAG,aAAa,aAAa,MAC5B,GAAG,aAAa,OAAO,KAAK,OAC7B,OAAO,WAAW,QAAQ,QAAQ,GAAG,EAAE,KAAK,KAC5C,GAAG,aAAa,MAAM,KACtB,IACA,UAAU,GAAG,EAAE;AAGjB,UAAI,CAAC,OAAO,WAAW,KAAK,GAAG,KAAK,IAAI,SAAS,GAAI;AAGrD,YAAM,MAAM,GAAG,IAAI,IAAI,IAAI,YAAY,CAAC;AACxC,UAAI,KAAK,IAAI,GAAG,EAAG;AACnB,WAAK,IAAI,GAAG;AAEZ,UAAI;AACJ,UAAI,QAAQ,SAAS;AACnB,cAAM,QAAQ;AACd,YAAI,MAAM,SAAS,WAAW,SAAS,KAAK,GAAG,EAAG,gBAAe;AAAA,iBACxD,MAAM,SAAS,cAAc,YAAY,KAAK,GAAG,EAAG,gBAAe;AAAA,iBACnE,MAAM,SAAS,SAAS,gBAAgB,KAAK,GAAG,EAAG,gBAAe;AAAA,iBAClE,MAAM,SAAS,UAAU,YAAY,KAAK,GAAG,EAAG,gBAAe;AAAA,iBAC/D,wBAAwB,KAAK,GAAG,EAAG,gBAAe;AAAA,iBAClD,sBAAsB,KAAK,GAAG,EAAG,gBAAe;AAAA,MAC3D;AAEA,UAAI,KAAK,EAAE,MAAM,OAAO,KAAK,cAAc,KAAK,aAAa,CAAC;AAAA,IAChE,CAAC;AAGD,UAAM,WAAW,CAAC,MAAqB;AACrC,UAAI,CAAC,SAAS,YAAY,QAAQ,OAAO,QAAQ,UAAU,UAAU,UAAU,EAAE,SAAS,EAAE,IAAI,EAAG,QAAO;AAC1G,UAAI,EAAE,SAAS,YAAY,EAAE,SAAS,SAAU,QAAO;AACvD,aAAO;AAAA,IACT;AACA,QAAI,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,CAAC;AAE5C,WAAO,IAAI,MAAM,GAAG,EAAE;AAAA,EACxB,CAAC;AACH;",
+  "sourcesContent": ["import type { Page } from 'playwright';\n\n/**\n * Reads a page as a structured outline (cheap + reliable), the way\n * Playwright MCP / the accessibility tree works \u2014 NOT by sending pixels to a\n * vision model. Vision is a fallback for image-only pages (added later).\n *\n * Returns the interactive elements found, which the Screen Reader Agent then\n * enriches with MEANING and saves to the Zeta Core.\n */\nexport interface RawElement {\n  role: string;\n  label: string;\n  selectorHint: string;\n  expectedData?: string;\n}\n\nexport async function readScreen(page: any): Promise<RawElement[]> {\n  return page.evaluate(() => {\n    // esbuild injects __name() for named arrow-function constants (e.g. `const priority = ...`).\n    // Polyfill prevents ReferenceError when the serialized callback runs in the browser.\n    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;\n    const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];\n    const seen = new Set<string>();\n\n    // Noise patterns to skip \u2014 cookie banners, footer chrome, version strings\n    // Intentionally excludes standalone \"close\" and \"copy\" \u2014 too many false positives\n    // (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets)\n    const noiseLabel =\n      /^(cookie|consent|privacy policy|terms of use|copyright|\u00A9|\\bv\\d+\\.\\d+|got it|dismiss|skip to|powered by|copied|\\s*$)/i;\n\n    // Include ARIA-role elements \u2014 Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags\n    const nodes = document.querySelectorAll(\n      'input:not([type=hidden]), select, textarea, button, a[href], ' +\n      '[role=\"button\"], [role=\"combobox\"], [role=\"slider\"], [role=\"switch\"], ' +\n      '[role=\"tab\"], [role=\"menuitem\"], [role=\"searchbox\"], [role=\"spinbutton\"]'\n    );\n\n    nodes.forEach((el) => {\n      const htmlEl = el as HTMLElement;\n\n      // Skip invisible elements\n      const style = window.getComputedStyle(htmlEl);\n      if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return;\n      const rect = el.getBoundingClientRect();\n      // Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links)\n      const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));\n      if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;\n\n      // Derive role\n      const tag = el.tagName.toLowerCase();\n      const role =\n        el.getAttribute('role') ||\n        (tag === 'input' ? ((el as HTMLInputElement).type || 'text') : tag);\n\n      // Derive the best human-readable label (prefer aria > placeholder > inner text)\n      const raw = (\n        el.getAttribute('aria-label') ||\n        el.getAttribute('placeholder') ||\n        (el.getAttribute('title') ?? '') ||\n        htmlEl.innerText?.replace(/\\s+/g, ' ').trim() ||\n        el.getAttribute('name') ||\n        ''\n      ).substring(0, 80);\n\n      // Skip empty, noise patterns, or very long labels (likely container text)\n      if (!raw || noiseLabel.test(raw) || raw.length > 70) return;\n\n      // Deduplicate by role+normalised label\n      const key = `${role}:${raw.toLowerCase()}`;\n      if (seen.has(key)) return;\n      seen.add(key);\n\n      let expectedData: string | undefined;\n      if (tag === 'input') {\n        const input = el as HTMLInputElement;\n        if (input.type === 'email' || /email/i.test(raw)) expectedData = 'valid_email';\n        else if (input.type === 'password' || /password/i.test(raw)) expectedData = 'password';\n        else if (input.type === 'tel' || /phone|mobile/i.test(raw)) expectedData = 'phone';\n        else if (input.type === 'date' || /\\bdate\\b/i.test(raw)) expectedData = 'date';\n        else if (/\\b(code|otp|token)\\b/i.test(raw)) expectedData = 'one_time_code';\n        else if (/\\b(search|query)\\b/i.test(raw)) expectedData = 'search_term';\n      }\n\n      out.push({ role, label: raw, selectorHint: tag, expectedData });\n    });\n\n    // Prioritise: form fields > buttons > links; within each group keep order\n    const priority = (e: typeof out[0]) => {\n      if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)) return 0;\n      if (e.role === 'button' || e.role === 'submit') return 1;\n      return 2;\n    };\n    out.sort((a, b) => priority(a) - priority(b));\n\n    return out.slice(0, 25); // cap at 25 \u2014 enough context, not overwhelming\n  });\n}\n"],
+  "mappings": "AAiBA,eAAsB,WAAW,MAAkC;AACjE,SAAO,KAAK,SAAS,MAAM;AAGzB,QAAI,OAAQ,WAAmB,WAAW,YAAa,CAAC,WAAmB,SAAS,CAAC,OAAY;AACjG,UAAM,MAAsF,CAAC;AAC7F,UAAM,OAAO,oBAAI,IAAY;AAK7B,UAAM,aACJ;AAGF,UAAM,QAAQ,SAAS;AAAA,MACrB;AAAA,IAGF;AAEA,UAAM,QAAQ,CAAC,OAAO;AACpB,YAAM,SAAS;AAGf,YAAM,QAAQ,OAAO,iBAAiB,MAAM;AAC5C,UAAI,MAAM,YAAY,UAAU,MAAM,eAAe,YAAY,MAAM,YAAY,IAAK;AACxF,YAAM,OAAO,GAAG,sBAAsB;AAEtC,YAAM,qBAAqB,CAAC,EAAE,GAAG,aAAa,YAAY,KAAK,GAAG,aAAa,iBAAiB;AAChG,UAAI,KAAK,UAAU,KAAK,KAAK,WAAW,KAAK,CAAC,mBAAoB;AAGlE,YAAM,MAAM,GAAG,QAAQ,YAAY;AACnC,YAAM,OACJ,GAAG,aAAa,MAAM,MACrB,QAAQ,UAAY,GAAwB,QAAQ,SAAU;AAGjE,YAAM,OACJ,GAAG,aAAa,YAAY,KAC5B,GAAG,aAAa,aAAa,MAC5B,GAAG,aAAa,OAAO,KAAK,OAC7B,OAAO,WAAW,QAAQ,QAAQ,GAAG,EAAE,KAAK,KAC5C,GAAG,aAAa,MAAM,KACtB,IACA,UAAU,GAAG,EAAE;AAGjB,UAAI,CAAC,OAAO,WAAW,KAAK,GAAG,KAAK,IAAI,SAAS,GAAI;AAGrD,YAAM,MAAM,GAAG,IAAI,IAAI,IAAI,YAAY,CAAC;AACxC,UAAI,KAAK,IAAI,GAAG,EAAG;AACnB,WAAK,IAAI,GAAG;AAEZ,UAAI;AACJ,UAAI,QAAQ,SAAS;AACnB,cAAM,QAAQ;AACd,YAAI,MAAM,SAAS,WAAW,SAAS,KAAK,GAAG,EAAG,gBAAe;AAAA,iBACxD,MAAM,SAAS,cAAc,YAAY,KAAK,GAAG,EAAG,gBAAe;AAAA,iBACnE,MAAM,SAAS,SAAS,gBAAgB,KAAK,GAAG,EAAG,gBAAe;AAAA,iBAClE,MAAM,SAAS,UAAU,YAAY,KAAK,GAAG,EAAG,gBAAe;AAAA,iBAC/D,wBAAwB,KAAK,GAAG,EAAG,gBAAe;AAAA,iBAClD,sBAAsB,KAAK,GAAG,EAAG,gBAAe;AAAA,MAC3D;AAEA,UAAI,KAAK,EAAE,MAAM,OAAO,KAAK,cAAc,KAAK,aAAa,CAAC;AAAA,IAChE,CAAC;AAGD,UAAM,WAAW,CAAC,MAAqB;AACrC,UAAI,CAAC,SAAS,YAAY,QAAQ,OAAO,QAAQ,UAAU,UAAU,UAAU,EAAE,SAAS,EAAE,IAAI,EAAG,QAAO;AAC1G,UAAI,EAAE,SAAS,YAAY,EAAE,SAAS,SAAU,QAAO;AACvD,aAAO;AAAA,IACT;AACA,QAAI,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,CAAC;AAE5C,WAAO,IAAI,MAAM,GAAG,EAAE;AAAA,EACxB,CAAC;AACH;",
   "names": []
 }

package/dist/service-worker-interceptor.js

@@ -0,0 +1,136 @@
+const EMPTY_RESULT = {
+  supported: false,
+  active: false,
+  registrations: [],
+  cacheStorages: [],
+  backgroundSync: [],
+  pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },
+  totalCachedEntries: 0
+};
+async function detectServiceWorkers(page) {
+  const result = await page.evaluate(async () => {
+    if (!("serviceWorker" in navigator)) return { supported: false, active: false, registrations: [] };
+    try {
+      const controller = navigator.serviceWorker.controller;
+      const regs = await navigator.serviceWorker.getRegistrations();
+      return {
+        supported: true,
+        active: !!controller,
+        registrations: regs.map((r) => ({
+          scope: r.scope,
+          scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,
+          state: r.active ? "active" : r.installing ? "installing" : r.waiting ? "waiting" : "none",
+          updateViaCache: r.updateViaCache
+        }))
+      };
+    } catch {
+      return { supported: true, active: false, registrations: [] };
+    }
+  });
+  return result;
+}
+async function enumerateCacheStorage(page) {
+  const caches = await page.evaluate(async () => {
+    if (!("caches" in window)) return [];
+    try {
+      const names = await window.caches.keys();
+      const result = [];
+      for (const name of names) {
+        try {
+          const cache = await window.caches.open(name);
+          const keys = await cache.keys();
+          result.push({
+            cacheName: name,
+            entryCount: keys.length,
+            sampleUrls: keys.slice(0, 15).map((k) => {
+              try {
+                return new URL(k.url).pathname;
+              } catch {
+                return k.url.slice(0, 80);
+              }
+            })
+          });
+        } catch {
+        }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return caches;
+}
+async function detectBackgroundSync(page) {
+  const syncInfo = await page.evaluate(async () => {
+    if (!("serviceWorker" in navigator)) return [];
+    try {
+      const regs = await navigator.serviceWorker.getRegistrations();
+      const result = [];
+      for (const reg of regs) {
+        if ("sync" in reg) {
+          try {
+            const tags = await reg.sync.getTags();
+            result.push({ scope: reg.scope, pendingTags: tags || [] });
+          } catch {
+            result.push({ scope: reg.scope, pendingTags: [] });
+          }
+        }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return syncInfo;
+}
+async function detectPwa(page) {
+  const pwa = await page.evaluate(() => {
+    const manifestEl = document.querySelector('link[rel="manifest"]');
+    const manifestUrl = manifestEl?.getAttribute("href") || null;
+    const hasSW = "serviceWorker" in navigator && !!navigator.serviceWorker.controller;
+    let hasPushPermission = false;
+    try {
+      hasPushPermission = Notification.permission === "granted";
+    } catch {
+    }
+    return {
+      isInstallable: !!(manifestUrl && hasSW),
+      manifestUrl,
+      hasPushPermission
+    };
+  });
+  return pwa;
+}
+async function inspectServiceWorker(page) {
+  try {
+    const url = page.url();
+    const isSecureContext = url.startsWith("https://") || /^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?/.test(url);
+    if (!isSecureContext) return { ...EMPTY_RESULT };
+    const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([
+      detectServiceWorkers(page),
+      enumerateCacheStorage(page),
+      detectBackgroundSync(page),
+      detectPwa(page)
+    ]);
+    const sw = swResult.status === "fulfilled" ? swResult.value : { supported: false, active: false, registrations: [] };
+    const cacheStorages = cacheResult.status === "fulfilled" ? cacheResult.value : [];
+    const backgroundSync = syncResult.status === "fulfilled" ? syncResult.value : [];
+    const pwa = pwaResult.status === "fulfilled" ? pwaResult.value : EMPTY_RESULT.pwa;
+    const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);
+    return {
+      supported: sw.supported,
+      active: sw.active,
+      registrations: sw.registrations,
+      cacheStorages,
+      backgroundSync,
+      pwa,
+      totalCachedEntries
+    };
+  } catch {
+    return { ...EMPTY_RESULT };
+  }
+}
+export {
+  inspectServiceWorker
+};
+//# sourceMappingURL=service-worker-interceptor.js.map

package/dist/service-worker-interceptor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/service-worker-interceptor.ts"],
+  "sourcesContent": ["/**\n * Service Worker detection \u2014 inspects SW registrations, cache storage, background\n * sync queues, and PWA installability for a crawled page.\n * Surfaces offline-first contracts and push-capable surfaces that HTTP-only\n * crawlers cannot observe.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface SwRegistration {\n  scope: string;\n  scriptURL: string | null;\n  state: 'active' | 'installing' | 'waiting' | 'none';\n  updateViaCache: string;\n}\n\nexport interface SwCacheEntry {\n  cacheName: string;\n  entryCount: number;\n  sampleUrls: string[];\n}\n\nexport interface BackgroundSyncInfo {\n  scope: string;\n  pendingTags: string[];\n}\n\nexport interface PwaInfo {\n  isInstallable: boolean;\n  manifestUrl: string | null;\n  hasPushPermission: boolean;\n}\n\nexport interface ServiceWorkerResult {\n  supported: boolean;\n  active: boolean;\n  registrations: SwRegistration[];\n  cacheStorages: SwCacheEntry[];\n  backgroundSync: BackgroundSyncInfo[];\n  pwa: PwaInfo;\n  totalCachedEntries: number;\n}\n\nconst EMPTY_RESULT: ServiceWorkerResult = {\n  supported: false,\n  active: false,\n  registrations: [],\n  cacheStorages: [],\n  backgroundSync: [],\n  pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },\n  totalCachedEntries: 0,\n};\n\nasync function detectServiceWorkers(\n  page: any,\n): Promise<{ supported: boolean; active: boolean; registrations: SwRegistration[] }> {\n  const result = await page.evaluate(async () => {\n    if (!('serviceWorker' in navigator)) return { supported: false, active: false, registrations: [] };\n    try {\n      const controller = navigator.serviceWorker.controller;\n      const regs = await navigator.serviceWorker.getRegistrations();\n      return {\n        supported: true,\n        active: !!controller,\n        registrations: regs.map(r => ({\n          scope: r.scope,\n          scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,\n          state: r.active ? 'active' : r.installing ? 'installing' : r.waiting ? 'waiting' : 'none',\n          updateViaCache: r.updateViaCache,\n        })),\n      };\n    } catch {\n      return { supported: true, active: false, registrations: [] };\n    }\n  });\n  return result as { supported: boolean; active: boolean; registrations: SwRegistration[] };\n}\n\nasync function enumerateCacheStorage(page: any): Promise<SwCacheEntry[]> {\n  const caches = await page.evaluate(async () => {\n    if (!('caches' in window)) return [];\n    try {\n      const names = await window.caches.keys();\n      const result = [];\n      for (const name of names) {\n        try {\n          const cache = await window.caches.open(name);\n          const keys = await cache.keys();\n          result.push({\n            cacheName: name,\n            entryCount: keys.length,\n            sampleUrls: keys.slice(0, 15).map(k => {\n              try { return new URL(k.url).pathname; } catch { return k.url.slice(0, 80); }\n            }),\n          });\n        } catch { /* skip inaccessible cache */ }\n      }\n      return result;\n    } catch {\n      return [];\n    }\n  });\n  return caches as SwCacheEntry[];\n}\n\nasync function detectBackgroundSync(page: any): Promise<BackgroundSyncInfo[]> {\n  const syncInfo = await page.evaluate(async () => {\n    if (!('serviceWorker' in navigator)) return [];\n    try {\n      const regs = await navigator.serviceWorker.getRegistrations();\n      const result = [];\n      for (const reg of regs) {\n        if ('sync' in reg) {\n          try {\n            const tags = await (reg as any).sync.getTags();\n            result.push({ scope: reg.scope, pendingTags: tags || [] });\n          } catch {\n            result.push({ scope: reg.scope, pendingTags: [] });\n          }\n        }\n      }\n      return result;\n    } catch {\n      return [];\n    }\n  });\n  return syncInfo as BackgroundSyncInfo[];\n}\n\nasync function detectPwa(page: any): Promise<PwaInfo> {\n  const pwa = await page.evaluate(() => {\n    const manifestEl = document.querySelector('link[rel=\"manifest\"]');\n    const manifestUrl = manifestEl?.getAttribute('href') || null;\n    const hasSW = 'serviceWorker' in navigator && !!navigator.serviceWorker.controller;\n    let hasPushPermission = false;\n    try { hasPushPermission = Notification.permission === 'granted'; } catch { /* ignore */ }\n    return {\n      isInstallable: !!(manifestUrl && hasSW),\n      manifestUrl,\n      hasPushPermission,\n    };\n  });\n  return pwa as PwaInfo;\n}\n\nexport async function inspectServiceWorker(page: any): Promise<ServiceWorkerResult> {\n  try {\n    // Allow SW on HTTPS or localhost (http://localhost and http://127.0.0.1 support SW)\n    const url = page.url();\n    const isSecureContext = url.startsWith('https://') ||\n      /^http:\\/\\/(localhost|127\\.0\\.0\\.1)(:\\d+)?/.test(url);\n    if (!isSecureContext) return { ...EMPTY_RESULT };\n\n    const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([\n      detectServiceWorkers(page),\n      enumerateCacheStorage(page),\n      detectBackgroundSync(page),\n      detectPwa(page),\n    ]);\n\n    const sw = swResult.status === 'fulfilled'\n      ? swResult.value\n      : { supported: false, active: false, registrations: [] };\n\n    const cacheStorages = cacheResult.status === 'fulfilled' ? cacheResult.value : [];\n    const backgroundSync = syncResult.status === 'fulfilled' ? syncResult.value : [];\n    const pwa = pwaResult.status === 'fulfilled'\n      ? pwaResult.value\n      : EMPTY_RESULT.pwa;\n\n    const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);\n\n    return {\n      supported: sw.supported,\n      active: sw.active,\n      registrations: sw.registrations,\n      cacheStorages,\n      backgroundSync,\n      pwa,\n      totalCachedEntries,\n    };\n  } catch {\n    return { ...EMPTY_RESULT };\n  }\n}\n"],
+  "mappings": "AA2CA,MAAM,eAAoC;AAAA,EACxC,WAAW;AAAA,EACX,QAAQ;AAAA,EACR,eAAe,CAAC;AAAA,EAChB,eAAe,CAAC;AAAA,EAChB,gBAAgB,CAAC;AAAA,EACjB,KAAK,EAAE,eAAe,OAAO,aAAa,MAAM,mBAAmB,MAAM;AAAA,EACzE,oBAAoB;AACtB;AAEA,eAAe,qBACb,MACmF;AACnF,QAAM,SAAS,MAAM,KAAK,SAAS,YAAY;AAC7C,QAAI,EAAE,mBAAmB,WAAY,QAAO,EAAE,WAAW,OAAO,QAAQ,OAAO,eAAe,CAAC,EAAE;AACjG,QAAI;AACF,YAAM,aAAa,UAAU,cAAc;AAC3C,YAAM,OAAO,MAAM,UAAU,cAAc,iBAAiB;AAC5D,aAAO;AAAA,QACL,WAAW;AAAA,QACX,QAAQ,CAAC,CAAC;AAAA,QACV,eAAe,KAAK,IAAI,QAAM;AAAA,UAC5B,OAAO,EAAE;AAAA,UACT,WAAW,EAAE,QAAQ,aAAa,EAAE,YAAY,aAAa,EAAE,SAAS,aAAa;AAAA,UACrF,OAAO,EAAE,SAAS,WAAW,EAAE,aAAa,eAAe,EAAE,UAAU,YAAY;AAAA,UACnF,gBAAgB,EAAE;AAAA,QACpB,EAAE;AAAA,MACJ;AAAA,IACF,QAAQ;AACN,aAAO,EAAE,WAAW,MAAM,QAAQ,OAAO,eAAe,CAAC,EAAE;AAAA,IAC7D;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,sBAAsB,MAAoC;AACvE,QAAM,SAAS,MAAM,KAAK,SAAS,YAAY;AAC7C,QAAI,EAAE,YAAY,QAAS,QAAO,CAAC;AACnC,QAAI;AACF,YAAM,QAAQ,MAAM,OAAO,OAAO,KAAK;AACvC,YAAM,SAAS,CAAC;AAChB,iBAAW,QAAQ,OAAO;AACxB,YAAI;AACF,gBAAM,QAAQ,MAAM,OAAO,OAAO,KAAK,IAAI;AAC3C,gBAAM,OAAO,MAAM,MAAM,KAAK;AAC9B,iBAAO,KAAK;AAAA,YACV,WAAW;AAAA,YACX,YAAY,KAAK;AAAA,YACjB,YAAY,KAAK,MAAM,GAAG,EAAE,EAAE,IAAI,OAAK;AACrC,kBAAI;AAAE,uBAAO,IAAI,IAAI,EAAE,GAAG,EAAE;AAAA,cAAU,QAAQ;AAAE,uBAAO,EAAE,IAAI,MAAM,GAAG,EAAE;AAAA,cAAG;AAAA,YAC7E,CAAC;AAAA,UACH,CAAC;AAAA,QACH,QAAQ;AAAA,QAAgC;AAAA,MAC1C;AACA,aAAO;AAAA,IACT,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,qBAAqB,MAA0C;AAC5E,QAAM,WAAW,MAAM,KAAK,SAAS,YAAY;AAC/C,QAAI,EAAE,mBAAmB,WAAY,QAAO,CAAC;AAC7C,QAAI;AACF,YAAM,OAAO,MAAM,UAAU,cAAc,iBAAiB;AAC5D,YAAM,SAAS,CAAC;AAChB,iBAAW,OAAO,MAAM;AACtB,YAAI,UAAU,KAAK;AACjB,cAAI;AACF,kBAAM,OAAO,MAAO,IAAY,KAAK,QAAQ;AAC7C,mBAAO,KAAK,EAAE,OAAO,IAAI,OAAO,aAAa,QAAQ,CAAC,EAAE,CAAC;AAAA,UAC3D,QAAQ;AACN,mBAAO,KAAK,EAAE,OAAO,IAAI,OAAO,aAAa,CAAC,EAAE,CAAC;AAAA,UACnD;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,UAAU,MAA6B;AACpD,QAAM,MAAM,MAAM,KAAK,SAAS,MAAM;AACpC,UAAM,aAAa,SAAS,cAAc,sBAAsB;AAChE,UAAM,cAAc,YAAY,aAAa,MAAM,KAAK;AACxD,UAAM,QAAQ,mBAAmB,aAAa,CAAC,CAAC,UAAU,cAAc;AACxE,QAAI,oBAAoB;AACxB,QAAI;AAAE,0BAAoB,aAAa,eAAe;AAAA,IAAW,QAAQ;AAAA,IAAe;AACxF,WAAO;AAAA,MACL,eAAe,CAAC,EAAE,eAAe;AAAA,MACjC;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAsB,qBAAqB,MAAyC;AAClF,MAAI;AAEF,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,kBAAkB,IAAI,WAAW,UAAU,KAC/C,4CAA4C,KAAK,GAAG;AACtD,QAAI,CAAC,gBAAiB,QAAO,EAAE,GAAG,aAAa;AAE/C,UAAM,CAAC,UAAU,aAAa,YAAY,SAAS,IAAI,MAAM,QAAQ,WAAW;AAAA,MAC9E,qBAAqB,IAAI;AAAA,MACzB,sBAAsB,IAAI;AAAA,MAC1B,qBAAqB,IAAI;AAAA,MACzB,UAAU,IAAI;AAAA,IAChB,CAAC;AAED,UAAM,KAAK,SAAS,WAAW,cAC3B,SAAS,QACT,EAAE,WAAW,OAAO,QAAQ,OAAO,eAAe,CAAC,EAAE;AAEzD,UAAM,gBAAgB,YAAY,WAAW,cAAc,YAAY,QAAQ,CAAC;AAChF,UAAM,iBAAiB,WAAW,WAAW,cAAc,WAAW,QAAQ,CAAC;AAC/E,UAAM,MAAM,UAAU,WAAW,cAC7B,UAAU,QACV,aAAa;AAEjB,UAAM,qBAAqB,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC;AAEjF,WAAO;AAAA,MACL,WAAW,GAAG;AAAA,MACd,QAAQ,GAAG;AAAA,MACX,eAAe,GAAG;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,QAAQ;AACN,WAAO,EAAE,GAAG,aAAa;AAAA,EAC3B;AACF;",
+  "names": []
+}

package/dist/session-capture.js

@@ -158,7 +158,7 @@ async function interactWithCurrentPage(page, isVisible, sleep) {
   }).catch(() => {
   });
 }
-async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts) {
+async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts, loginInstructions) {
   const browser = await chromium.launch({ headless: true });
   try {
     const contextOpts = {};
@@ -201,7 +201,9 @@ async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts)
           sh,
           { username: creds.username, password: creds.password, getOneTimeCode },
           appUrl,
-          captchaOpts
+          captchaOpts,
+          void 0,
+          loginInstructions
         );
         if (shLoggedIn) {
           const shCtx = sh.context ?? sh.browserContext;

package/dist/session-capture.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/session-capture.ts"],
-  "sourcesContent": ["import { chromium } from 'playwright';\nimport { randomBytes } from 'node:crypto';\nimport { login } from './auth.js';\nimport { generateTOTP } from './job-queue.js';\n\ninterface CaptureSession {\n  browser: any;\n  context: any;\n  timer: ReturnType<typeof setTimeout>;\n}\n\nconst activeSessions = new Map<string, CaptureSession>();\n\n/**\n * Opens a headed (visible) browser at appUrl and returns a captureId.\n * The user logs in manually, then calls completeSessionCapture() to harvest storageState.\n * Sessions expire after 10 minutes if not completed.\n *\n * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.\n * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.\n */\nexport async function startSessionCapture(appUrl: string): Promise<string> {\n  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';\n  const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;\n  if (inDocker || noDisplay) {\n    throw new Error('HEADLESS_ENV');\n  }\n\n  const id = randomBytes(16).toString('hex');\n  const browser = await chromium.launch({ headless: false });\n  const context = await browser.newContext();\n  const page = await context.newPage();\n  await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => {});\n  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {}), 10 * 60 * 1000);\n  activeSessions.set(id, { browser, context, timer });\n  return id;\n}\n\n/** Captures storageState from the open browser, closes it, and returns the JSON string. */\nexport async function completeSessionCapture(captureId: string): Promise<string> {\n  const session = activeSessions.get(captureId);\n  if (!session) throw new Error('Capture session not found or expired.');\n  clearTimeout(session.timer);\n  const state = await session.context.storageState();\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n  return JSON.stringify(state);\n}\n\n/** Closes the browser without capturing \u2014 called on Cancel. */\nexport async function cancelSessionCapture(captureId: string): Promise<void> {\n  const session = activeSessions.get(captureId);\n  if (!session) return;\n  clearTimeout(session.timer);\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n}\n\n/**\n * Post-login exploration: navigates nav links, cycles tabs, advances carousels,\n * opens accordions, scrolls \u2014 so the browser accumulates full session state before capture.\n * All best-effort: never throws, never fails the capture.\n */\nasync function exploreAppAfterLogin(page: any): Promise<void> {\n  const sleep = (ms: number) => page.waitForTimeout(ms).catch(() => {});\n\n  const isVisible = `\n    function isVisible(el) {\n      const s = window.getComputedStyle(el);\n      if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;\n      const r = el.getBoundingClientRect();\n      return r.width > 4 && r.height > 4;\n    }\n  `;\n\n  try {\n    const origin = await page.evaluate(() => location.origin).catch(() => '');\n    const startUrl = await page.evaluate(() => location.href).catch(() => '');\n\n    // \u2500\u2500 1. Expand collapsed nav menus / hamburgers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await page.evaluate(new Function(`\n      ${isVisible}\n      const inBadCtx = el => {\n        let p = el.parentElement;\n        while (p) {\n          const role = p.getAttribute('role') ?? '';\n          if (['dialog','alertdialog','combobox','listbox'].includes(role)) return true;\n          p = p.parentElement;\n        }\n        return false;\n      };\n      const targets = new Set();\n      document.querySelectorAll('[aria-haspopup]:not([aria-expanded=\"true\"])').forEach(el => {\n        if (isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[aria-expanded=\"false\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[data-state=\"closed\"],[data-headlessui-state=\"closed\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      targets.forEach(el => { try { el.click(); } catch {} });\n    `) as () => void).catch(() => {});\n    await sleep(600);\n\n    // \u2500\u2500 2. Collect nav links from the expanded DOM \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    const navLinks: string[] = await page.evaluate(new Function(`\n      ${isVisible}\n      const links = new Set();\n      const navEls = document.querySelectorAll('nav a, [role=\"navigation\"] a, aside a, header a');\n      navEls.forEach(a => {\n        const href = a.getAttribute('href');\n        if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;\n        if (!isVisible(a)) return;\n        try {\n          const url = new URL(href, location.href);\n          if (url.origin === location.origin) links.add(url.href);\n        } catch {}\n      });\n      return [...links].slice(0, 10); // cap at 10 nav pages\n    `) as () => string[]).catch(() => []);\n\n    // Dismiss opened menus before navigating\n    await page.keyboard?.press('Escape').catch(() => {});\n    await sleep(200);\n\n    // \u2500\u2500 3. Visit each nav link, interact, return \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    for (const link of navLinks) {\n      try {\n        if (link === startUrl) continue;\n        await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n        await sleep(400);\n        await interactWithCurrentPage(page, isVisible, sleep);\n      } catch { /* skip broken page */ }\n    }\n\n    // Return to start page\n    if (navLinks.length > 0) {\n      await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n      await sleep(400);\n    }\n\n    // \u2500\u2500 4. Final interaction pass on landing page \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await interactWithCurrentPage(page, isVisible, sleep);\n\n    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});\n  } catch {\n    // Exploration is best-effort\n  }\n}\n\n/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */\nasync function interactWithCurrentPage(page: any, isVisible: string, sleep: (ms: number) => Promise<void>): Promise<void> {\n  // Cycle through tabs\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const tabs = [...document.querySelectorAll('[role=\"tab\"]')].filter(isVisible);\n    tabs.forEach(tab => { try { tab.click(); } catch {} });\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Advance carousels (next buttons)\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const nextBtns = [...document.querySelectorAll(\n      'button[aria-label*=\"next\" i], button[aria-label*=\"forward\" i], button[aria-label*=\"right\" i], ' +\n      '[class*=\"carousel\" i] button, [class*=\"slider\" i] button, [data-direction=\"next\"]'\n    )].filter(isVisible);\n    // Click next up to 3 times per carousel\n    for (let i = 0; i < 3; i++) {\n      nextBtns.forEach(btn => { try { btn.click(); } catch {} });\n    }\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Open closed accordions / disclosure panels\n  await page.evaluate(new Function(`\n    ${isVisible}\n    document.querySelectorAll(\n      '[data-state=\"closed\"] button, [aria-expanded=\"false\"], ' +\n      'details:not([open]) > summary, [class*=\"accordion\" i] button'\n    ).forEach(el => {\n      if (isVisible(el)) { try { el.click(); } catch {} }\n    });\n  `) as () => void).catch(() => {});\n  await sleep(300);\n\n  // Scroll to bottom then back to top\n  await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => {});\n  await sleep(300);\n  await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => {});\n}\n\n/**\n * Headless login + session capture \u2014 works on Docker/cloud (no display needed).\n * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,\n * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).\n *\n * Returns { storageState, cookies, localStorage } on success.\n * Throws a human-readable error on failure.\n */\nexport async function validateAndCaptureSession(\n  appUrl: string,\n  creds: {\n    username: string; password: string; totpSecret?: string;\n    httpBasicUsername?: string; httpBasicPassword?: string;\n  },\n  llmConfig?: { modelName: string; apiKey: string },\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {\n  const browser = await chromium.launch({ headless: true });\n  try {\n    const contextOpts: any = {};\n    if (creds.httpBasicUsername && creds.httpBasicPassword) {\n      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };\n    }\n    const context = await browser.newContext(contextOpts);\n    const page = await context.newPage();\n\n    const getOneTimeCode = creds.totpSecret\n      ? async () => generateTOTP(creds.totpSecret!)\n      : undefined;\n\n    let loggedIn: boolean;\n    try {\n      loggedIn = await login(page, appUrl, {\n        username: creds.username,\n        password: creds.password,\n        getOneTimeCode,\n      });\n    } catch (loginErr: any) {\n      // login() throws when it can read a real error message from the page\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      throw Object.assign(loginErr, {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n      });\n    }\n\n    // Stagehand AI fallback when DOM login fails and LLM config is available\n    if (!loggedIn && llmConfig) {\n      let sh: any = null;\n      try {\n        const { Stagehand } = await import('@browserbasehq/stagehand');\n        const { loginWithStagehand } = await import('./ai-login.js');\n        sh = new Stagehand({\n          env: 'LOCAL',\n          model: { modelName: llmConfig.modelName as any, apiKey: llmConfig.apiKey },\n          serverCache: false,\n          verbose: 0,\n          localBrowserLaunchOptions: { headless: true },\n        });\n        await sh.init();\n        const shPage = await (sh as any).resolvePage();\n        await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});\n        const shLoggedIn = await loginWithStagehand(\n          sh,\n          { username: creds.username, password: creds.password, getOneTimeCode },\n          appUrl,\n          captchaOpts,\n        );\n        if (shLoggedIn) {\n          const shCtx = (sh as any).context ?? (sh as any).browserContext;\n          if (shCtx) {\n            const shState = await shCtx.storageState();\n            const shCookies = shState.cookies?.length ?? 0;\n            const shLocalStorage = shState.origins?.reduce((s: number, o: any) => s + (o.localStorage?.length ?? 0), 0) ?? 0;\n            if (shCookies > 0 || shLocalStorage > 0) {\n              await sh.close().catch(() => {});\n              sh = null;\n              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };\n            }\n          }\n          loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below\n        }\n      } catch (shErr: any) {\n        const msg = String(shErr?.message ?? shErr);\n        if (msg.includes('CAPTCHA') || shErr?.isSSOError) throw shErr;\n        // Stagehand failed silently \u2014 fall through to \"credentials rejected\"\n      } finally {\n        if (sh) await sh.close().catch(() => {});\n      }\n    }\n\n    if (!loggedIn) {\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      const challenge = await page.evaluate(() => {\n        const text = (document.body?.innerText ?? '').toLowerCase();\n        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||\n          !!document.querySelector('iframe[src*=\"challenges.cloudflare.com\"], .cf-turnstile, [data-sitekey]');\n        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||\n          !!document.querySelector('iframe[src*=\"recaptcha\"], iframe[src*=\"hcaptcha\"], #captcha, .captcha, [id*=\"captcha\"], [class*=\"captcha\"]');\n        return { hasCloudflare, hasCaptcha };\n      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));\n      const message = challenge.hasCloudflare\n        ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'\n        : challenge.hasCaptcha\n          ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'\n          : 'Login failed \u2014 credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';\n      const err = Object.assign(new Error(message), {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n        reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',\n      });\n      throw err;\n    }\n\n    // Wait for post-login navigation to settle before capturing session\n    await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});\n\n    // Capture post-login landing URL before exploration navigates away\n    const finalUrl = page.url();\n\n    // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state\n    await exploreAppAfterLogin(page);\n\n    const state = await context.storageState();\n    const cookies = state.cookies?.length ?? 0;\n    const localStorage = state.origins?.reduce(\n      (sum, o) => sum + (o.localStorage?.length ?? 0), 0\n    ) ?? 0;\n\n    if (cookies === 0 && localStorage === 0) {\n      throw new Error('Login appeared to succeed but no session data was captured \u2014 try manual session capture instead');\n    }\n\n    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };\n  } finally {\n    await browser.close().catch(() => {});\n  }\n}\n"],
-  "mappings": "AAAA,SAAS,gBAAgB;AACzB,SAAS,mBAAmB;AAC5B,SAAS,aAAa;AACtB,SAAS,oBAAoB;AAQ7B,MAAM,iBAAiB,oBAAI,IAA4B;AAUvD,eAAsB,oBAAoB,QAAiC;AACzE,QAAM,WAAW,QAAQ,IAAI,yBAAyB;AACtD,QAAM,YAAY,QAAQ,aAAa,WAAW,CAAC,QAAQ,IAAI;AAC/D,MAAI,YAAY,WAAW;AACzB,UAAM,IAAI,MAAM,cAAc;AAAA,EAChC;AAEA,QAAM,KAAK,YAAY,EAAE,EAAE,SAAS,KAAK;AACzC,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,MAAM,CAAC;AACzD,QAAM,UAAU,MAAM,QAAQ,WAAW;AACzC,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,QAAM,KAAK,KAAK,QAAQ,EAAE,WAAW,mBAAmB,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACzE,QAAM,QAAQ,WAAW,MAAM,qBAAqB,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC,GAAG,KAAK,KAAK,GAAI;AACvF,iBAAe,IAAI,IAAI,EAAE,SAAS,SAAS,MAAM,CAAC;AAClD,SAAO;AACT;AAGA,eAAsB,uBAAuB,WAAoC;AAC/E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,uCAAuC;AACrE,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,MAAM,QAAQ,QAAQ,aAAa;AACjD,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AAC/B,SAAO,KAAK,UAAU,KAAK;AAC7B;AAGA,eAAsB,qBAAqB,WAAkC;AAC3E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS;AACd,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AACjC;AAOA,eAAe,qBAAqB,MAA0B;AAC5D,QAAM,QAAQ,CAAC,OAAe,KAAK,eAAe,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAEpE,QAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASlB,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,MAAM,SAAS,MAAM,EAAE,MAAM,MAAM,EAAE;AACxE,UAAM,WAAW,MAAM,KAAK,SAAS,MAAM,SAAS,IAAI,EAAE,MAAM,MAAM,EAAE;AAGxE,UAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAqBZ,CAAe,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAChC,UAAM,MAAM,GAAG;AAGf,UAAM,WAAqB,MAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QACxD,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAaZ,CAAmB,EAAE,MAAM,MAAM,CAAC,CAAC;AAGpC,UAAM,KAAK,UAAU,MAAM,QAAQ,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACnD,UAAM,MAAM,GAAG;AAGf,eAAW,QAAQ,UAAU;AAC3B,UAAI;AACF,YAAI,SAAS,SAAU;AACvB,cAAM,KAAK,KAAK,MAAM,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACnF,cAAM,MAAM,GAAG;AACf,cAAM,wBAAwB,MAAM,WAAW,KAAK;AAAA,MACtD,QAAQ;AAAA,MAAyB;AAAA,IACnC;AAGA,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,KAAK,KAAK,UAAU,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACvF,YAAM,MAAM,GAAG;AAAA,IACjB;AAGA,UAAM,wBAAwB,MAAM,WAAW,KAAK;AAEpD,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC/E,QAAQ;AAAA,EAER;AACF;AAGA,eAAe,wBAAwB,MAAW,WAAmB,OAAqD;AAExH,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA,GAGZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GASZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,SAAS,KAAK,cAAc,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACxH,QAAM,MAAM,GAAG;AACf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,GAAG,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACjG;AAUA,eAAsB,0BACpB,QACA,OAIA,WACA,aAC6F;AAC7F,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AACxD,MAAI;AACF,UAAM,cAAmB,CAAC;AAC1B,QAAI,MAAM,qBAAqB,MAAM,mBAAmB;AACtD,kBAAY,kBAAkB,EAAE,UAAU,MAAM,mBAAmB,UAAU,MAAM,kBAAkB;AAAA,IACvG;AACA,UAAM,UAAU,MAAM,QAAQ,WAAW,WAAW;AACpD,UAAM,OAAO,MAAM,QAAQ,QAAQ;AAEnC,UAAM,iBAAiB,MAAM,aACzB,YAAY,aAAa,MAAM,UAAW,IAC1C;AAEJ,QAAI;AACJ,QAAI;AACF,iBAAW,MAAM,MAAM,MAAM,QAAQ;AAAA,QACnC,UAAU,MAAM;AAAA,QAChB,UAAU,MAAM;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,SAAS,UAAe;AAEtB,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,OAAO,OAAO,UAAU;AAAA,QAC5B,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,MAC5D,CAAC;AAAA,IACH;AAGA,QAAI,CAAC,YAAY,WAAW;AAC1B,UAAI,KAAU;AACd,UAAI;AACF,cAAM,EAAE,UAAU,IAAI,MAAM,OAAO,0BAA0B;AAC7D,cAAM,EAAE,mBAAmB,IAAI,MAAM,OAAO,eAAe;AAC3D,aAAK,IAAI,UAAU;AAAA,UACjB,KAAK;AAAA,UACL,OAAO,EAAE,WAAW,UAAU,WAAkB,QAAQ,UAAU,OAAO;AAAA,UACzE,aAAa;AAAA,UACb,SAAS;AAAA,UACT,2BAA2B,EAAE,UAAU,KAAK;AAAA,QAC9C,CAAC;AACD,cAAM,GAAG,KAAK;AACd,cAAM,SAAS,MAAO,GAAW,YAAY;AAC7C,cAAM,OAAO,KAAK,QAAQ,EAAE,WAAW,eAAe,SAAS,IAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACvF,cAAM,aAAa,MAAM;AAAA,UACvB;AAAA,UACA,EAAE,UAAU,MAAM,UAAU,UAAU,MAAM,UAAU,eAAe;AAAA,UACrE;AAAA,UACA;AAAA,QACF;AACA,YAAI,YAAY;AACd,gBAAM,QAAS,GAAW,WAAY,GAAW;AACjD,cAAI,OAAO;AACT,kBAAM,UAAU,MAAM,MAAM,aAAa;AACzC,kBAAM,YAAY,QAAQ,SAAS,UAAU;AAC7C,kBAAM,iBAAiB,QAAQ,SAAS,OAAO,CAAC,GAAW,MAAW,KAAK,EAAE,cAAc,UAAU,IAAI,CAAC,KAAK;AAC/G,gBAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,oBAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,cAAC,CAAC;AAC/B,mBAAK;AACL,qBAAO,EAAE,cAAc,KAAK,UAAU,OAAO,GAAG,SAAS,WAAW,cAAc,eAAe;AAAA,YACnG;AAAA,UACF;AACA,qBAAW;AAAA,QACb;AAAA,MACF,SAAS,OAAY;AACnB,cAAM,MAAM,OAAO,OAAO,WAAW,KAAK;AAC1C,YAAI,IAAI,SAAS,SAAS,KAAK,OAAO,WAAY,OAAM;AAAA,MAE1D,UAAE;AACA,YAAI,GAAI,OAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACzC;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,YAAY,MAAM,KAAK,SAAS,MAAM;AAC1C,cAAM,QAAQ,SAAS,MAAM,aAAa,IAAI,YAAY;AAC1D,cAAM,gBAAgB,+EAA+E,KAAK,IAAI,KAC5G,CAAC,CAAC,SAAS,cAAc,yEAAyE;AACpG,cAAM,aAAa,iBAAiB,6BAA6B,KAAK,IAAI,KACxE,CAAC,CAAC,SAAS,cAAc,4GAA4G;AACvI,eAAO,EAAE,eAAe,WAAW;AAAA,MACrC,CAAC,EAAE,MAAM,OAAO,EAAE,eAAe,OAAO,YAAY,MAAM,EAAE;AAC5D,YAAM,UAAU,UAAU,gBACtB,8MACA,UAAU,aACR,oHACA;AACN,YAAM,MAAM,OAAO,OAAO,IAAI,MAAM,OAAO,GAAG;AAAA,QAC5C,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,QAC1D,QAAQ,UAAU,gBAAgB,yBAAyB,UAAU,aAAa,sBAAsB;AAAA,MAC1G,CAAC;AACD,YAAM;AAAA,IACR;AAGA,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7E,UAAM,WAAW,KAAK,IAAI;AAG1B,UAAM,qBAAqB,IAAI;AAE/B,UAAM,QAAQ,MAAM,QAAQ,aAAa;AACzC,UAAM,UAAU,MAAM,SAAS,UAAU;AACzC,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC,CAAC,KAAK,MAAM,OAAO,EAAE,cAAc,UAAU;AAAA,MAAI;AAAA,IACnD,KAAK;AAEL,QAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,YAAM,IAAI,MAAM,sGAAiG;AAAA,IACnH;AAEA,WAAO,EAAE,cAAc,KAAK,UAAU,KAAK,GAAG,SAAS,cAAc,SAAS;AAAA,EAChF,UAAE;AACA,UAAM,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACtC;AACF;",
+  "sourcesContent": ["import { chromium } from 'playwright';\nimport { randomBytes } from 'node:crypto';\nimport { login } from './auth.js';\nimport { generateTOTP } from './job-queue.js';\n\ninterface CaptureSession {\n  browser: any;\n  context: any;\n  timer: ReturnType<typeof setTimeout>;\n}\n\nconst activeSessions = new Map<string, CaptureSession>();\n\n/**\n * Opens a headed (visible) browser at appUrl and returns a captureId.\n * The user logs in manually, then calls completeSessionCapture() to harvest storageState.\n * Sessions expire after 10 minutes if not completed.\n *\n * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.\n * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.\n */\nexport async function startSessionCapture(appUrl: string): Promise<string> {\n  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';\n  const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;\n  if (inDocker || noDisplay) {\n    throw new Error('HEADLESS_ENV');\n  }\n\n  const id = randomBytes(16).toString('hex');\n  const browser = await chromium.launch({ headless: false });\n  const context = await browser.newContext();\n  const page = await context.newPage();\n  await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => {});\n  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {}), 10 * 60 * 1000);\n  activeSessions.set(id, { browser, context, timer });\n  return id;\n}\n\n/** Captures storageState from the open browser, closes it, and returns the JSON string. */\nexport async function completeSessionCapture(captureId: string): Promise<string> {\n  const session = activeSessions.get(captureId);\n  if (!session) throw new Error('Capture session not found or expired.');\n  clearTimeout(session.timer);\n  const state = await session.context.storageState();\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n  return JSON.stringify(state);\n}\n\n/** Closes the browser without capturing \u2014 called on Cancel. */\nexport async function cancelSessionCapture(captureId: string): Promise<void> {\n  const session = activeSessions.get(captureId);\n  if (!session) return;\n  clearTimeout(session.timer);\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n}\n\n/**\n * Post-login exploration: navigates nav links, cycles tabs, advances carousels,\n * opens accordions, scrolls \u2014 so the browser accumulates full session state before capture.\n * All best-effort: never throws, never fails the capture.\n */\nasync function exploreAppAfterLogin(page: any): Promise<void> {\n  const sleep = (ms: number) => page.waitForTimeout(ms).catch(() => {});\n\n  const isVisible = `\n    function isVisible(el) {\n      const s = window.getComputedStyle(el);\n      if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;\n      const r = el.getBoundingClientRect();\n      return r.width > 4 && r.height > 4;\n    }\n  `;\n\n  try {\n    const origin = await page.evaluate(() => location.origin).catch(() => '');\n    const startUrl = await page.evaluate(() => location.href).catch(() => '');\n\n    // \u2500\u2500 1. Expand collapsed nav menus / hamburgers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await page.evaluate(new Function(`\n      ${isVisible}\n      const inBadCtx = el => {\n        let p = el.parentElement;\n        while (p) {\n          const role = p.getAttribute('role') ?? '';\n          if (['dialog','alertdialog','combobox','listbox'].includes(role)) return true;\n          p = p.parentElement;\n        }\n        return false;\n      };\n      const targets = new Set();\n      document.querySelectorAll('[aria-haspopup]:not([aria-expanded=\"true\"])').forEach(el => {\n        if (isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[aria-expanded=\"false\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[data-state=\"closed\"],[data-headlessui-state=\"closed\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      targets.forEach(el => { try { el.click(); } catch {} });\n    `) as () => void).catch(() => {});\n    await sleep(600);\n\n    // \u2500\u2500 2. Collect nav links from the expanded DOM \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    const navLinks: string[] = await page.evaluate(new Function(`\n      ${isVisible}\n      const links = new Set();\n      const navEls = document.querySelectorAll('nav a, [role=\"navigation\"] a, aside a, header a');\n      navEls.forEach(a => {\n        const href = a.getAttribute('href');\n        if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;\n        if (!isVisible(a)) return;\n        try {\n          const url = new URL(href, location.href);\n          if (url.origin === location.origin) links.add(url.href);\n        } catch {}\n      });\n      return [...links].slice(0, 10); // cap at 10 nav pages\n    `) as () => string[]).catch(() => []);\n\n    // Dismiss opened menus before navigating\n    await page.keyboard?.press('Escape').catch(() => {});\n    await sleep(200);\n\n    // \u2500\u2500 3. Visit each nav link, interact, return \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    for (const link of navLinks) {\n      try {\n        if (link === startUrl) continue;\n        await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n        await sleep(400);\n        await interactWithCurrentPage(page, isVisible, sleep);\n      } catch { /* skip broken page */ }\n    }\n\n    // Return to start page\n    if (navLinks.length > 0) {\n      await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n      await sleep(400);\n    }\n\n    // \u2500\u2500 4. Final interaction pass on landing page \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await interactWithCurrentPage(page, isVisible, sleep);\n\n    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});\n  } catch {\n    // Exploration is best-effort\n  }\n}\n\n/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */\nasync function interactWithCurrentPage(page: any, isVisible: string, sleep: (ms: number) => Promise<void>): Promise<void> {\n  // Cycle through tabs\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const tabs = [...document.querySelectorAll('[role=\"tab\"]')].filter(isVisible);\n    tabs.forEach(tab => { try { tab.click(); } catch {} });\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Advance carousels (next buttons)\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const nextBtns = [...document.querySelectorAll(\n      'button[aria-label*=\"next\" i], button[aria-label*=\"forward\" i], button[aria-label*=\"right\" i], ' +\n      '[class*=\"carousel\" i] button, [class*=\"slider\" i] button, [data-direction=\"next\"]'\n    )].filter(isVisible);\n    // Click next up to 3 times per carousel\n    for (let i = 0; i < 3; i++) {\n      nextBtns.forEach(btn => { try { btn.click(); } catch {} });\n    }\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Open closed accordions / disclosure panels\n  await page.evaluate(new Function(`\n    ${isVisible}\n    document.querySelectorAll(\n      '[data-state=\"closed\"] button, [aria-expanded=\"false\"], ' +\n      'details:not([open]) > summary, [class*=\"accordion\" i] button'\n    ).forEach(el => {\n      if (isVisible(el)) { try { el.click(); } catch {} }\n    });\n  `) as () => void).catch(() => {});\n  await sleep(300);\n\n  // Scroll to bottom then back to top\n  await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => {});\n  await sleep(300);\n  await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => {});\n}\n\n/**\n * Headless login + session capture \u2014 works on Docker/cloud (no display needed).\n * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,\n * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).\n *\n * Returns { storageState, cookies, localStorage } on success.\n * Throws a human-readable error on failure.\n */\nexport async function validateAndCaptureSession(\n  appUrl: string,\n  creds: {\n    username: string; password: string; totpSecret?: string;\n    httpBasicUsername?: string; httpBasicPassword?: string;\n  },\n  llmConfig?: { modelName: string; apiKey: string },\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n  loginInstructions?: string,\n): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {\n  const browser = await chromium.launch({ headless: true });\n  try {\n    const contextOpts: any = {};\n    if (creds.httpBasicUsername && creds.httpBasicPassword) {\n      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };\n    }\n    const context = await browser.newContext(contextOpts);\n    const page = await context.newPage();\n\n    const getOneTimeCode = creds.totpSecret\n      ? async () => generateTOTP(creds.totpSecret!)\n      : undefined;\n\n    let loggedIn: boolean;\n    try {\n      loggedIn = await login(page, appUrl, {\n        username: creds.username,\n        password: creds.password,\n        getOneTimeCode,\n      });\n    } catch (loginErr: any) {\n      // login() throws when it can read a real error message from the page\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      throw Object.assign(loginErr, {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n      });\n    }\n\n    // Stagehand AI fallback when DOM login fails and LLM config is available\n    if (!loggedIn && llmConfig) {\n      let sh: any = null;\n      try {\n        const { Stagehand } = await import('@browserbasehq/stagehand');\n        const { loginWithStagehand } = await import('./ai-login.js');\n        sh = new Stagehand({\n          env: 'LOCAL',\n          model: { modelName: llmConfig.modelName as any, apiKey: llmConfig.apiKey },\n          serverCache: false,\n          verbose: 0,\n          localBrowserLaunchOptions: { headless: true },\n        });\n        await sh.init();\n        const shPage = await (sh as any).resolvePage();\n        await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});\n        const shLoggedIn = await loginWithStagehand(\n          sh,\n          { username: creds.username, password: creds.password, getOneTimeCode },\n          appUrl,\n          captchaOpts,\n          undefined,\n          loginInstructions,\n        );\n        if (shLoggedIn) {\n          const shCtx = (sh as any).context ?? (sh as any).browserContext;\n          if (shCtx) {\n            const shState = await shCtx.storageState();\n            const shCookies = shState.cookies?.length ?? 0;\n            const shLocalStorage = shState.origins?.reduce((s: number, o: any) => s + (o.localStorage?.length ?? 0), 0) ?? 0;\n            if (shCookies > 0 || shLocalStorage > 0) {\n              await sh.close().catch(() => {});\n              sh = null;\n              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };\n            }\n          }\n          loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below\n        }\n      } catch (shErr: any) {\n        const msg = String(shErr?.message ?? shErr);\n        if (msg.includes('CAPTCHA') || shErr?.isSSOError) throw shErr;\n        // Stagehand failed silently \u2014 fall through to \"credentials rejected\"\n      } finally {\n        if (sh) await sh.close().catch(() => {});\n      }\n    }\n\n    if (!loggedIn) {\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      const challenge = await page.evaluate(() => {\n        const text = (document.body?.innerText ?? '').toLowerCase();\n        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||\n          !!document.querySelector('iframe[src*=\"challenges.cloudflare.com\"], .cf-turnstile, [data-sitekey]');\n        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||\n          !!document.querySelector('iframe[src*=\"recaptcha\"], iframe[src*=\"hcaptcha\"], #captcha, .captcha, [id*=\"captcha\"], [class*=\"captcha\"]');\n        return { hasCloudflare, hasCaptcha };\n      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));\n      const message = challenge.hasCloudflare\n        ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'\n        : challenge.hasCaptcha\n          ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'\n          : 'Login failed \u2014 credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';\n      const err = Object.assign(new Error(message), {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n        reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',\n      });\n      throw err;\n    }\n\n    // Wait for post-login navigation to settle before capturing session\n    await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});\n\n    // Capture post-login landing URL before exploration navigates away\n    const finalUrl = page.url();\n\n    // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state\n    await exploreAppAfterLogin(page);\n\n    const state = await context.storageState();\n    const cookies = state.cookies?.length ?? 0;\n    const localStorage = state.origins?.reduce(\n      (sum, o) => sum + (o.localStorage?.length ?? 0), 0\n    ) ?? 0;\n\n    if (cookies === 0 && localStorage === 0) {\n      throw new Error('Login appeared to succeed but no session data was captured \u2014 try manual session capture instead');\n    }\n\n    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };\n  } finally {\n    await browser.close().catch(() => {});\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,gBAAgB;AACzB,SAAS,mBAAmB;AAC5B,SAAS,aAAa;AACtB,SAAS,oBAAoB;AAQ7B,MAAM,iBAAiB,oBAAI,IAA4B;AAUvD,eAAsB,oBAAoB,QAAiC;AACzE,QAAM,WAAW,QAAQ,IAAI,yBAAyB;AACtD,QAAM,YAAY,QAAQ,aAAa,WAAW,CAAC,QAAQ,IAAI;AAC/D,MAAI,YAAY,WAAW;AACzB,UAAM,IAAI,MAAM,cAAc;AAAA,EAChC;AAEA,QAAM,KAAK,YAAY,EAAE,EAAE,SAAS,KAAK;AACzC,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,MAAM,CAAC;AACzD,QAAM,UAAU,MAAM,QAAQ,WAAW;AACzC,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,QAAM,KAAK,KAAK,QAAQ,EAAE,WAAW,mBAAmB,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACzE,QAAM,QAAQ,WAAW,MAAM,qBAAqB,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC,GAAG,KAAK,KAAK,GAAI;AACvF,iBAAe,IAAI,IAAI,EAAE,SAAS,SAAS,MAAM,CAAC;AAClD,SAAO;AACT;AAGA,eAAsB,uBAAuB,WAAoC;AAC/E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,uCAAuC;AACrE,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,MAAM,QAAQ,QAAQ,aAAa;AACjD,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AAC/B,SAAO,KAAK,UAAU,KAAK;AAC7B;AAGA,eAAsB,qBAAqB,WAAkC;AAC3E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS;AACd,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AACjC;AAOA,eAAe,qBAAqB,MAA0B;AAC5D,QAAM,QAAQ,CAAC,OAAe,KAAK,eAAe,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAEpE,QAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASlB,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,MAAM,SAAS,MAAM,EAAE,MAAM,MAAM,EAAE;AACxE,UAAM,WAAW,MAAM,KAAK,SAAS,MAAM,SAAS,IAAI,EAAE,MAAM,MAAM,EAAE;AAGxE,UAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAqBZ,CAAe,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAChC,UAAM,MAAM,GAAG;AAGf,UAAM,WAAqB,MAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QACxD,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAaZ,CAAmB,EAAE,MAAM,MAAM,CAAC,CAAC;AAGpC,UAAM,KAAK,UAAU,MAAM,QAAQ,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACnD,UAAM,MAAM,GAAG;AAGf,eAAW,QAAQ,UAAU;AAC3B,UAAI;AACF,YAAI,SAAS,SAAU;AACvB,cAAM,KAAK,KAAK,MAAM,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACnF,cAAM,MAAM,GAAG;AACf,cAAM,wBAAwB,MAAM,WAAW,KAAK;AAAA,MACtD,QAAQ;AAAA,MAAyB;AAAA,IACnC;AAGA,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,KAAK,KAAK,UAAU,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACvF,YAAM,MAAM,GAAG;AAAA,IACjB;AAGA,UAAM,wBAAwB,MAAM,WAAW,KAAK;AAEpD,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC/E,QAAQ;AAAA,EAER;AACF;AAGA,eAAe,wBAAwB,MAAW,WAAmB,OAAqD;AAExH,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA,GAGZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GASZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,SAAS,KAAK,cAAc,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACxH,QAAM,MAAM,GAAG;AACf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,GAAG,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACjG;AAUA,eAAsB,0BACpB,QACA,OAIA,WACA,aACA,mBAC6F;AAC7F,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AACxD,MAAI;AACF,UAAM,cAAmB,CAAC;AAC1B,QAAI,MAAM,qBAAqB,MAAM,mBAAmB;AACtD,kBAAY,kBAAkB,EAAE,UAAU,MAAM,mBAAmB,UAAU,MAAM,kBAAkB;AAAA,IACvG;AACA,UAAM,UAAU,MAAM,QAAQ,WAAW,WAAW;AACpD,UAAM,OAAO,MAAM,QAAQ,QAAQ;AAEnC,UAAM,iBAAiB,MAAM,aACzB,YAAY,aAAa,MAAM,UAAW,IAC1C;AAEJ,QAAI;AACJ,QAAI;AACF,iBAAW,MAAM,MAAM,MAAM,QAAQ;AAAA,QACnC,UAAU,MAAM;AAAA,QAChB,UAAU,MAAM;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,SAAS,UAAe;AAEtB,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,OAAO,OAAO,UAAU;AAAA,QAC5B,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,MAC5D,CAAC;AAAA,IACH;AAGA,QAAI,CAAC,YAAY,WAAW;AAC1B,UAAI,KAAU;AACd,UAAI;AACF,cAAM,EAAE,UAAU,IAAI,MAAM,OAAO,0BAA0B;AAC7D,cAAM,EAAE,mBAAmB,IAAI,MAAM,OAAO,eAAe;AAC3D,aAAK,IAAI,UAAU;AAAA,UACjB,KAAK;AAAA,UACL,OAAO,EAAE,WAAW,UAAU,WAAkB,QAAQ,UAAU,OAAO;AAAA,UACzE,aAAa;AAAA,UACb,SAAS;AAAA,UACT,2BAA2B,EAAE,UAAU,KAAK;AAAA,QAC9C,CAAC;AACD,cAAM,GAAG,KAAK;AACd,cAAM,SAAS,MAAO,GAAW,YAAY;AAC7C,cAAM,OAAO,KAAK,QAAQ,EAAE,WAAW,eAAe,SAAS,IAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACvF,cAAM,aAAa,MAAM;AAAA,UACvB;AAAA,UACA,EAAE,UAAU,MAAM,UAAU,UAAU,MAAM,UAAU,eAAe;AAAA,UACrE;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,YAAI,YAAY;AACd,gBAAM,QAAS,GAAW,WAAY,GAAW;AACjD,cAAI,OAAO;AACT,kBAAM,UAAU,MAAM,MAAM,aAAa;AACzC,kBAAM,YAAY,QAAQ,SAAS,UAAU;AAC7C,kBAAM,iBAAiB,QAAQ,SAAS,OAAO,CAAC,GAAW,MAAW,KAAK,EAAE,cAAc,UAAU,IAAI,CAAC,KAAK;AAC/G,gBAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,oBAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,cAAC,CAAC;AAC/B,mBAAK;AACL,qBAAO,EAAE,cAAc,KAAK,UAAU,OAAO,GAAG,SAAS,WAAW,cAAc,eAAe;AAAA,YACnG;AAAA,UACF;AACA,qBAAW;AAAA,QACb;AAAA,MACF,SAAS,OAAY;AACnB,cAAM,MAAM,OAAO,OAAO,WAAW,KAAK;AAC1C,YAAI,IAAI,SAAS,SAAS,KAAK,OAAO,WAAY,OAAM;AAAA,MAE1D,UAAE;AACA,YAAI,GAAI,OAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACzC;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,YAAY,MAAM,KAAK,SAAS,MAAM;AAC1C,cAAM,QAAQ,SAAS,MAAM,aAAa,IAAI,YAAY;AAC1D,cAAM,gBAAgB,+EAA+E,KAAK,IAAI,KAC5G,CAAC,CAAC,SAAS,cAAc,yEAAyE;AACpG,cAAM,aAAa,iBAAiB,6BAA6B,KAAK,IAAI,KACxE,CAAC,CAAC,SAAS,cAAc,4GAA4G;AACvI,eAAO,EAAE,eAAe,WAAW;AAAA,MACrC,CAAC,EAAE,MAAM,OAAO,EAAE,eAAe,OAAO,YAAY,MAAM,EAAE;AAC5D,YAAM,UAAU,UAAU,gBACtB,8MACA,UAAU,aACR,oHACA;AACN,YAAM,MAAM,OAAO,OAAO,IAAI,MAAM,OAAO,GAAG;AAAA,QAC5C,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,QAC1D,QAAQ,UAAU,gBAAgB,yBAAyB,UAAU,aAAa,sBAAsB;AAAA,MAC1G,CAAC;AACD,YAAM;AAAA,IACR;AAGA,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7E,UAAM,WAAW,KAAK,IAAI;AAG1B,UAAM,qBAAqB,IAAI;AAE/B,UAAM,QAAQ,MAAM,QAAQ,aAAa;AACzC,UAAM,UAAU,MAAM,SAAS,UAAU;AACzC,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC,CAAC,KAAK,MAAM,OAAO,EAAE,cAAc,UAAU;AAAA,MAAI;AAAA,IACnD,KAAK;AAEL,QAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,YAAM,IAAI,MAAM,sGAAiG;AAAA,IACnH;AAEA,WAAO,EAAE,cAAc,KAAK,UAAU,KAAK,GAAG,SAAS,cAAc,SAAS;AAAA,EAChF,UAAE;AACA,UAAM,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACtC;AACF;",
   "names": []
 }

package/dist/shadow-dom-crawler.js

@@ -0,0 +1,132 @@
+async function enumerateShadowHosts(page) {
+  const raw = await page.evaluate(() => {
+    if (typeof globalThis.__name === "undefined") globalThis.__name = (fn) => fn;
+    function collectHosts(root, depth) {
+      if (depth > 8) return [];
+      const hosts = [];
+      const els = root.querySelectorAll("*");
+      for (const el of Array.from(els)) {
+        const htmlEl = el;
+        if (htmlEl.shadowRoot) {
+          const interactive = htmlEl.shadowRoot.querySelectorAll(
+            'button,input,select,textarea,a[href],[role="button"],[role="link"],[role="checkbox"],[role="menuitem"],[role="tab"],[role="switch"]'
+          );
+          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll("slot")).map(
+            (s) => s.name || "default"
+          );
+          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll("*")).filter(
+            (c) => c.shadowRoot
+          ).length;
+          hosts.push({
+            tagName: htmlEl.tagName.toLowerCase(),
+            hostId: htmlEl.id || htmlEl.className?.toString()?.split(" ")[0] || "",
+            shadowMode: "open",
+            depth,
+            interactiveCount: interactive.length,
+            interactiveDetails: Array.from(interactive).slice(0, 8).map((i) => {
+              const iEl = i;
+              return {
+                role: iEl.getAttribute("role") || iEl.tagName.toLowerCase(),
+                label: (iEl.getAttribute("aria-label") || iEl.getAttribute("placeholder") || iEl.textContent?.trim() || "").slice(0, 40),
+                type: iEl.getAttribute("type") || "",
+                tagName: iEl.tagName.toLowerCase()
+              };
+            }),
+            slotNames: slots,
+            childShadowHosts: nestedHosts
+          });
+          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);
+          hosts.push(...nested);
+        } else if (el.tagName.includes("-") && !el.shadowRoot) {
+          hosts.push({
+            tagName: el.tagName.toLowerCase(),
+            hostId: el.id || "",
+            shadowMode: "closed",
+            depth,
+            interactiveCount: 0,
+            interactiveDetails: [],
+            slotNames: [],
+            childShadowHosts: 0
+          });
+        }
+      }
+      return hosts;
+    }
+    return collectHosts(document, 0);
+  });
+  return raw.slice(0, 100).map((h) => ({
+    tagName: h.tagName,
+    hostId: h.hostId,
+    shadowMode: h.shadowMode,
+    depth: h.depth,
+    interactiveCount: h.interactiveCount,
+    interactiveElements: h.interactiveDetails,
+    slotNames: h.slotNames,
+    childShadowHosts: h.childShadowHosts
+  }));
+}
+async function detectWebComponentLibrary(page) {
+  return page.evaluate(() => {
+    const checks = [
+      ["sl-button", "Shoelace"],
+      ["fast-button", "Microsoft FAST"],
+      ["mwc-button", "Material Web Components"],
+      ["ion-button", "Ionic"],
+      ["vaadin-button", "Vaadin"],
+      ["fluent-button", "Fluent UI Web Components"],
+      ["hy-icon", "Hydra"]
+    ];
+    const detected = [];
+    for (const [tag, name] of checks) {
+      if (customElements.get(tag)) detected.push(name);
+    }
+    const allTags = Array.from(document.querySelectorAll("*")).map((e) => e.tagName.toLowerCase()).filter((t) => t.includes("-"));
+    const uniqueTags = Array.from(new Set(allTags));
+    if (uniqueTags.some((t) => t.startsWith("sl-")) && !detected.includes("Shoelace"))
+      detected.push("Shoelace");
+    if (uniqueTags.some((t) => t.startsWith("fast-")) && !detected.includes("Microsoft FAST"))
+      detected.push("Microsoft FAST");
+    if (uniqueTags.some((t) => t.startsWith("ion-")) && !detected.includes("Ionic"))
+      detected.push("Ionic");
+    return detected;
+  });
+}
+async function crawlShadowDom(page) {
+  const empty = {
+    shadowHostCount: 0,
+    openShadowCount: 0,
+    closedShadowCount: 0,
+    maxNestingDepth: 0,
+    hosts: [],
+    totalInteractiveInShadow: 0,
+    detectedLibraries: [],
+    blindSpots: []
+  };
+  try {
+    const [hosts, detectedLibraries] = await Promise.all([
+      enumerateShadowHosts(page),
+      detectWebComponentLibrary(page)
+    ]);
+    const openHosts = hosts.filter((h) => h.shadowMode === "open");
+    const closedHosts = hosts.filter((h) => h.shadowMode === "closed");
+    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);
+    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);
+    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];
+    return {
+      shadowHostCount: hosts.length,
+      openShadowCount: openHosts.length,
+      closedShadowCount: closedHosts.length,
+      maxNestingDepth,
+      hosts,
+      totalInteractiveInShadow,
+      detectedLibraries,
+      blindSpots
+    };
+  } catch {
+    return empty;
+  }
+}
+export {
+  crawlShadowDom
+};
+//# sourceMappingURL=shadow-dom-crawler.js.map

package/dist/shadow-dom-crawler.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/shadow-dom-crawler.ts"],
+  "sourcesContent": ["/**\n * Shadow DOM crawler \u2014 enumerate shadow hosts and detect web component libraries.\n *\n * Walks the live DOM recursively (up to depth 8) to find all open shadow roots\n * and custom elements with closed shadow roots, then classifies the component\n * framework in use. Results feed ZeTa's test generator so it can account for\n * elements that are invisible to standard Playwright locators.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface ShadowInteractiveEl {\n  role: string;\n  label: string;\n  type: string;\n  tagName: string;\n}\n\nexport interface ShadowHostInfo {\n  tagName: string;\n  hostId: string;\n  shadowMode: 'open' | 'closed';\n  depth: number;\n  interactiveCount: number;\n  interactiveElements: ShadowInteractiveEl[];\n  slotNames: string[];\n  childShadowHosts: number;\n}\n\nexport interface ShadowDomCrawlResult {\n  shadowHostCount: number;\n  openShadowCount: number;\n  closedShadowCount: number;\n  maxNestingDepth: number;\n  hosts: ShadowHostInfo[];\n  totalInteractiveInShadow: number;\n  detectedLibraries: string[];\n  blindSpots: string[];\n}\n\ninterface RawShadowHost {\n  tagName: string;\n  hostId: string;\n  shadowMode: 'open' | 'closed';\n  depth: number;\n  interactiveCount: number;\n  interactiveDetails: Array<{ role: string; label: string; type: string; tagName: string }>;\n  slotNames: string[];\n  childShadowHosts: number;\n}\n\nasync function enumerateShadowHosts(page: any): Promise<ShadowHostInfo[]> {\n  const raw: RawShadowHost[] = await page.evaluate(() => {\n    // esbuild injects __name() for named function declarations; polyfill prevents ReferenceError in browser.\n    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;\n    function collectHosts(root: Document | ShadowRoot, depth: number): RawShadowHost[] {\n      if (depth > 8) return [];\n      const hosts: RawShadowHost[] = [];\n      const els = root.querySelectorAll('*');\n      for (const el of Array.from(els)) {\n        const htmlEl = el as HTMLElement & { shadowRoot: ShadowRoot | null };\n        if (htmlEl.shadowRoot) {\n          const interactive = htmlEl.shadowRoot.querySelectorAll(\n            'button,input,select,textarea,a[href],[role=\"button\"],[role=\"link\"],[role=\"checkbox\"],[role=\"menuitem\"],[role=\"tab\"],[role=\"switch\"]'\n          );\n          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll('slot')).map(\n            (s) => (s as HTMLSlotElement).name || 'default'\n          );\n          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll('*')).filter(\n            (c) => (c as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot\n          ).length;\n          hosts.push({\n            tagName: htmlEl.tagName.toLowerCase(),\n            hostId:\n              htmlEl.id ||\n              (htmlEl.className?.toString()?.split(' ')[0]) ||\n              '',\n            shadowMode: 'open',\n            depth,\n            interactiveCount: interactive.length,\n            interactiveDetails: Array.from(interactive)\n              .slice(0, 8)\n              .map((i) => {\n                const iEl = i as HTMLElement;\n                return {\n                  role: iEl.getAttribute('role') || iEl.tagName.toLowerCase(),\n                  label: (\n                    iEl.getAttribute('aria-label') ||\n                    iEl.getAttribute('placeholder') ||\n                    iEl.textContent?.trim() ||\n                    ''\n                  ).slice(0, 40),\n                  type: iEl.getAttribute('type') || '',\n                  tagName: iEl.tagName.toLowerCase(),\n                };\n              }),\n            slotNames: slots,\n            childShadowHosts: nestedHosts,\n          });\n          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);\n          hosts.push(...nested);\n        } else if (el.tagName.includes('-') && !(el as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot) {\n          hosts.push({\n            tagName: el.tagName.toLowerCase(),\n            hostId: (el as HTMLElement).id || '',\n            shadowMode: 'closed',\n            depth,\n            interactiveCount: 0,\n            interactiveDetails: [],\n            slotNames: [],\n            childShadowHosts: 0,\n          });\n        }\n      }\n      return hosts;\n    }\n    return collectHosts(document, 0);\n  });\n\n  return raw.slice(0, 100).map((h) => ({\n    tagName: h.tagName,\n    hostId: h.hostId,\n    shadowMode: h.shadowMode,\n    depth: h.depth,\n    interactiveCount: h.interactiveCount,\n    interactiveElements: h.interactiveDetails,\n    slotNames: h.slotNames,\n    childShadowHosts: h.childShadowHosts,\n  }));\n}\n\nasync function detectWebComponentLibrary(page: any): Promise<string[]> {\n  return page.evaluate(() => {\n    const checks: [string, string][] = [\n      ['sl-button', 'Shoelace'],\n      ['fast-button', 'Microsoft FAST'],\n      ['mwc-button', 'Material Web Components'],\n      ['ion-button', 'Ionic'],\n      ['vaadin-button', 'Vaadin'],\n      ['fluent-button', 'Fluent UI Web Components'],\n      ['hy-icon', 'Hydra'],\n    ];\n    const detected: string[] = [];\n    for (const [tag, name] of checks) {\n      if (customElements.get(tag)) detected.push(name);\n    }\n    const allTags = Array.from(document.querySelectorAll('*'))\n      .map((e) => e.tagName.toLowerCase())\n      .filter((t) => t.includes('-'));\n    const uniqueTags = Array.from(new Set(allTags));\n    if (uniqueTags.some((t) => t.startsWith('sl-')) && !detected.includes('Shoelace'))\n      detected.push('Shoelace');\n    if (uniqueTags.some((t) => t.startsWith('fast-')) && !detected.includes('Microsoft FAST'))\n      detected.push('Microsoft FAST');\n    if (uniqueTags.some((t) => t.startsWith('ion-')) && !detected.includes('Ionic'))\n      detected.push('Ionic');\n    return detected;\n  });\n}\n\nexport async function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult> {\n  const empty: ShadowDomCrawlResult = {\n    shadowHostCount: 0,\n    openShadowCount: 0,\n    closedShadowCount: 0,\n    maxNestingDepth: 0,\n    hosts: [],\n    totalInteractiveInShadow: 0,\n    detectedLibraries: [],\n    blindSpots: [],\n  };\n\n  try {\n    const [hosts, detectedLibraries] = await Promise.all([\n      enumerateShadowHosts(page),\n      detectWebComponentLibrary(page),\n    ]);\n\n    const openHosts = hosts.filter((h) => h.shadowMode === 'open');\n    const closedHosts = hosts.filter((h) => h.shadowMode === 'closed');\n    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);\n    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);\n    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];\n\n    return {\n      shadowHostCount: hosts.length,\n      openShadowCount: openHosts.length,\n      closedShadowCount: closedHosts.length,\n      maxNestingDepth,\n      hosts,\n      totalInteractiveInShadow,\n      detectedLibraries,\n      blindSpots,\n    };\n  } catch {\n    return empty;\n  }\n}\n"],
+  "mappings": "AAmDA,eAAe,qBAAqB,MAAsC;AACxE,QAAM,MAAuB,MAAM,KAAK,SAAS,MAAM;AAErD,QAAI,OAAQ,WAAmB,WAAW,YAAa,CAAC,WAAmB,SAAS,CAAC,OAAY;AACjG,aAAS,aAAa,MAA6B,OAAgC;AACjF,UAAI,QAAQ,EAAG,QAAO,CAAC;AACvB,YAAM,QAAyB,CAAC;AAChC,YAAM,MAAM,KAAK,iBAAiB,GAAG;AACrC,iBAAW,MAAM,MAAM,KAAK,GAAG,GAAG;AAChC,cAAM,SAAS;AACf,YAAI,OAAO,YAAY;AACrB,gBAAM,cAAc,OAAO,WAAW;AAAA,YACpC;AAAA,UACF;AACA,gBAAM,QAAQ,MAAM,KAAK,OAAO,WAAW,iBAAiB,MAAM,CAAC,EAAE;AAAA,YACnE,CAAC,MAAO,EAAsB,QAAQ;AAAA,UACxC;AACA,gBAAM,cAAc,MAAM,KAAK,OAAO,WAAW,iBAAiB,GAAG,CAAC,EAAE;AAAA,YACtE,CAAC,MAAO,EAAsD;AAAA,UAChE,EAAE;AACF,gBAAM,KAAK;AAAA,YACT,SAAS,OAAO,QAAQ,YAAY;AAAA,YACpC,QACE,OAAO,MACN,OAAO,WAAW,SAAS,GAAG,MAAM,GAAG,EAAE,CAAC,KAC3C;AAAA,YACF,YAAY;AAAA,YACZ;AAAA,YACA,kBAAkB,YAAY;AAAA,YAC9B,oBAAoB,MAAM,KAAK,WAAW,EACvC,MAAM,GAAG,CAAC,EACV,IAAI,CAAC,MAAM;AACV,oBAAM,MAAM;AACZ,qBAAO;AAAA,gBACL,MAAM,IAAI,aAAa,MAAM,KAAK,IAAI,QAAQ,YAAY;AAAA,gBAC1D,QACE,IAAI,aAAa,YAAY,KAC7B,IAAI,aAAa,aAAa,KAC9B,IAAI,aAAa,KAAK,KACtB,IACA,MAAM,GAAG,EAAE;AAAA,gBACb,MAAM,IAAI,aAAa,MAAM,KAAK;AAAA,gBAClC,SAAS,IAAI,QAAQ,YAAY;AAAA,cACnC;AAAA,YACF,CAAC;AAAA,YACH,WAAW;AAAA,YACX,kBAAkB;AAAA,UACpB,CAAC;AACD,gBAAM,SAAS,aAAa,OAAO,YAAY,QAAQ,CAAC;AACxD,gBAAM,KAAK,GAAG,MAAM;AAAA,QACtB,WAAW,GAAG,QAAQ,SAAS,GAAG,KAAK,CAAE,GAAuD,YAAY;AAC1G,gBAAM,KAAK;AAAA,YACT,SAAS,GAAG,QAAQ,YAAY;AAAA,YAChC,QAAS,GAAmB,MAAM;AAAA,YAClC,YAAY;AAAA,YACZ;AAAA,YACA,kBAAkB;AAAA,YAClB,oBAAoB,CAAC;AAAA,YACrB,WAAW,CAAC;AAAA,YACZ,kBAAkB;AAAA,UACpB,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AACA,WAAO,aAAa,UAAU,CAAC;AAAA,EACjC,CAAC;AAED,SAAO,IAAI,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,OAAO;AAAA,IACnC,SAAS,EAAE;AAAA,IACX,QAAQ,EAAE;AAAA,IACV,YAAY,EAAE;AAAA,IACd,OAAO,EAAE;AAAA,IACT,kBAAkB,EAAE;AAAA,IACpB,qBAAqB,EAAE;AAAA,IACvB,WAAW,EAAE;AAAA,IACb,kBAAkB,EAAE;AAAA,EACtB,EAAE;AACJ;AAEA,eAAe,0BAA0B,MAA8B;AACrE,SAAO,KAAK,SAAS,MAAM;AACzB,UAAM,SAA6B;AAAA,MACjC,CAAC,aAAa,UAAU;AAAA,MACxB,CAAC,eAAe,gBAAgB;AAAA,MAChC,CAAC,cAAc,yBAAyB;AAAA,MACxC,CAAC,cAAc,OAAO;AAAA,MACtB,CAAC,iBAAiB,QAAQ;AAAA,MAC1B,CAAC,iBAAiB,0BAA0B;AAAA,MAC5C,CAAC,WAAW,OAAO;AAAA,IACrB;AACA,UAAM,WAAqB,CAAC;AAC5B,eAAW,CAAC,KAAK,IAAI,KAAK,QAAQ;AAChC,UAAI,eAAe,IAAI,GAAG,EAAG,UAAS,KAAK,IAAI;AAAA,IACjD;AACA,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,GAAG,CAAC,EACtD,IAAI,CAAC,MAAM,EAAE,QAAQ,YAAY,CAAC,EAClC,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,CAAC;AAChC,UAAM,aAAa,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC;AAC9C,QAAI,WAAW,KAAK,CAAC,MAAM,EAAE,WAAW,KAAK,CAAC,KAAK,CAAC,SAAS,SAAS,UAAU;AAC9E,eAAS,KAAK,UAAU;AAC1B,QAAI,WAAW,KAAK,CAAC,MAAM,EAAE,WAAW,OAAO,CAAC,KAAK,CAAC,SAAS,SAAS,gBAAgB;AACtF,eAAS,KAAK,gBAAgB;AAChC,QAAI,WAAW,KAAK,CAAC,MAAM,EAAE,WAAW,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,OAAO;AAC5E,eAAS,KAAK,OAAO;AACvB,WAAO;AAAA,EACT,CAAC;AACH;AAEA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,QAA8B;AAAA,IAClC,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,mBAAmB;AAAA,IACnB,iBAAiB;AAAA,IACjB,OAAO,CAAC;AAAA,IACR,0BAA0B;AAAA,IAC1B,mBAAmB,CAAC;AAAA,IACpB,YAAY,CAAC;AAAA,EACf;AAEA,MAAI;AACF,UAAM,CAAC,OAAO,iBAAiB,IAAI,MAAM,QAAQ,IAAI;AAAA,MACnD,qBAAqB,IAAI;AAAA,MACzB,0BAA0B,IAAI;AAAA,IAChC,CAAC;AAED,UAAM,YAAY,MAAM,OAAO,CAAC,MAAM,EAAE,eAAe,MAAM;AAC7D,UAAM,cAAc,MAAM,OAAO,CAAC,MAAM,EAAE,eAAe,QAAQ;AACjE,UAAM,kBAAkB,MAAM,OAAO,CAAC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,KAAK,GAAG,CAAC;AAC1E,UAAM,2BAA2B,UAAU,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,kBAAkB,CAAC;AACzF,UAAM,aAAa,CAAC,GAAG,IAAI,IAAI,YAAY,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjE,WAAO;AAAA,MACL,iBAAiB,MAAM;AAAA,MACvB,iBAAiB,UAAU;AAAA,MAC3B,mBAAmB,YAAY;AAAA,MAC/B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/sitemap.js

@@ -5,7 +5,11 @@ import { logger } from "./logger.js";
 const FETCH_TIMEOUT_MS = 1e4;
 const MAX_SITEMAP_INDEX_DEPTH = 3;
 const MAX_SUB_SITEMAPS = 50;
-const MAX_URLS = 5e3;
+async function getSitemapMaxUrls() {
+  const { getSystemConfig } = await import("@detiq/core");
+  const cfg = await getSystemConfig("crawl_tuning");
+  return parseInt(String(cfg?.sitemapMaxUrls ?? "5000"), 10);
+}
 function toArray(x) {
   if (x == null) return [];
   return Array.isArray(x) ? x : [x];
@@ -21,7 +25,7 @@ function sameSite(candidateUrl, siteOrigin) {
     return false;
   }
 }
-function extractUrlsFromParsedSitemap(parsed, origin) {
+function extractUrlsFromParsedSitemap(parsed, origin, maxUrls = 5e3) {
   const rawPageUrls = [];
   let subSitemapUrls = [];
   if (parsed?.sitemapindex) {
@@ -33,7 +37,7 @@ function extractUrlsFromParsedSitemap(parsed, origin) {
     );
   }
   const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
-  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };
 }
 async function fetchAndParseXml(url, dispatcher) {
   try {
@@ -54,19 +58,20 @@ async function fetchAndParseXml(url, dispatcher) {
     return null;
   }
 }
-async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
+async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher, maxUrls) {
   if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  if (maxUrls === void 0) maxUrls = await getSitemapMaxUrls();
   const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
   if (!parsed) return [];
-  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);
   const collected = [...pageUrls];
-  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {
     const subResults = await Promise.all(
-      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))
     );
     for (const sub of subResults) collected.push(...sub);
   }
-  return [...new Set(collected)].slice(0, MAX_URLS);
+  return [...new Set(collected)].slice(0, maxUrls);
 }
 async function discoverSitemapUrls(appUrl, sitemapUrl) {
   let origin;
@@ -115,6 +120,7 @@ async function discoverProjectUrls(appUrl, opts) {
     return { candidates: [], sitemapFound: false };
   }
   const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  const maxUrls = await getSitemapMaxUrls();
   let robotsSitemaps = [];
   try {
     assertCrawlUrlSafe(`${origin}/robots.txt`);
@@ -135,12 +141,12 @@ async function discoverProjectUrls(appUrl, opts) {
   const seen = /* @__PURE__ */ new Map();
   let sitemapFound = false;
   for (const sm of sitemapTargets) {
-    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);
     if (urls.length) sitemapFound = true;
     const source = robotsSet.has(sm) ? "robots" : "sitemap";
     for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
   }
-  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };
 }
 export {
   buildProxyDispatcher,