@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/screenshot-masker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/screenshot-masker.ts"],
+  "sourcesContent": ["/**\n * Screenshot masker \u2014 blur or black-box PII regions in captured screenshots.\n *\n * Strategy: inject a CSS overlay into the page before screenshot capture that\n * covers detected PII-likely elements (input[type=email], input[type=password],\n * elements with PII-containing text) with a blur or solid black mask.\n *\n * This is a CSS-based approach (no image processing library needed):\n * 1. Identify likely-PII DOM elements via selectors\n * 2. Add an overlaid <div> with blur/black filter at each element's bounding box\n * 3. Take screenshot\n * 4. Remove overlays\n *\n * Falls back gracefully if elements are not found or overlay injection fails.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface MaskingOpts {\n  /** Additional CSS selectors to mask (in addition to defaults) */\n  extraSelectors?: string[];\n  /** Masking style: 'blur' or 'blackout' */\n  style?: 'blur' | 'blackout';\n  /** Blur strength in px (default 8) */\n  blurPx?: number;\n}\n\n// Default PII-sensitive selectors\nconst DEFAULT_PII_SELECTORS = [\n  'input[type=\"email\"]',\n  'input[type=\"password\"]',\n  'input[type=\"tel\"]',\n  'input[autocomplete=\"cc-number\"]',\n  'input[autocomplete=\"cc-csc\"]',\n  'input[autocomplete=\"ssn\"]',\n  '[data-pii=\"true\"]',\n  '[data-sensitive=\"true\"]',\n];\n\nexport async function maskScreenshot(\n  page: Page,\n  opts: MaskingOpts = {},\n): Promise<{ maskedCount: number; screenshot: Buffer }> {\n  const { extraSelectors = [], style = 'blur', blurPx = 8 } = opts;\n  const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];\n\n  let maskedCount = 0;\n  const overlayIds: string[] = [];\n\n  // Inject CSS overlays\n  try {\n    const result = await page.evaluate(({ selectors, style, blurPx }) => {\n      const ids: string[] = [];\n      for (const sel of selectors) {\n        try {\n          const elements = Array.from(document.querySelectorAll(sel));\n          for (const el of elements) {\n            const rect = el.getBoundingClientRect();\n            if (rect.width === 0 || rect.height === 0) continue;\n            const overlay = document.createElement('div');\n            const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;\n            overlay.id = id;\n            overlay.style.cssText = [\n              'position: fixed',\n              `top: ${rect.top + window.scrollY}px`,\n              `left: ${rect.left + window.scrollX}px`,\n              `width: ${rect.width}px`,\n              `height: ${rect.height}px`,\n              'z-index: 2147483647',\n              style === 'blur'\n                ? `backdrop-filter: blur(${blurPx}px); -webkit-backdrop-filter: blur(${blurPx}px); background: rgba(0,0,0,0.1);`\n                : 'background: #000000;',\n              'pointer-events: none',\n            ].join('; ');\n            document.body.appendChild(overlay);\n            ids.push(id);\n          }\n        } catch { /* selector error */ }\n      }\n      return ids;\n    }, { selectors, style, blurPx });\n\n    overlayIds.push(...result);\n    maskedCount = result.length;\n  } catch { /* non-fatal */ }\n\n  // Take screenshot with overlays active\n  const screenshot = await page.screenshot({ type: 'png', fullPage: false });\n\n  // Remove overlays\n  if (overlayIds.length > 0) {\n    await page.evaluate((ids) => {\n      for (const id of ids) document.getElementById(id)?.remove();\n    }, overlayIds).catch(() => {});\n  }\n\n  return { maskedCount, screenshot };\n}\n"],
+  "mappings": "AA4BA,MAAM,wBAAwB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,eAAsB,eACpB,MACA,OAAoB,CAAC,GACiC;AACtD,QAAM,EAAE,iBAAiB,CAAC,GAAG,QAAQ,QAAQ,SAAS,EAAE,IAAI;AAC5D,QAAM,YAAY,CAAC,GAAG,uBAAuB,GAAG,cAAc;AAE9D,MAAI,cAAc;AAClB,QAAM,aAAuB,CAAC;AAG9B,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,CAAC,EAAE,WAAAA,YAAW,OAAAC,QAAO,QAAAC,QAAO,MAAM;AACnE,YAAM,MAAgB,CAAC;AACvB,iBAAW,OAAOF,YAAW;AAC3B,YAAI;AACF,gBAAM,WAAW,MAAM,KAAK,SAAS,iBAAiB,GAAG,CAAC;AAC1D,qBAAW,MAAM,UAAU;AACzB,kBAAM,OAAO,GAAG,sBAAsB;AACtC,gBAAI,KAAK,UAAU,KAAK,KAAK,WAAW,EAAG;AAC3C,kBAAM,UAAU,SAAS,cAAc,KAAK;AAC5C,kBAAM,KAAK,eAAe,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AAC7D,oBAAQ,KAAK;AACb,oBAAQ,MAAM,UAAU;AAAA,cACtB;AAAA,cACA,QAAQ,KAAK,MAAM,OAAO,OAAO;AAAA,cACjC,SAAS,KAAK,OAAO,OAAO,OAAO;AAAA,cACnC,UAAU,KAAK,KAAK;AAAA,cACpB,WAAW,KAAK,MAAM;AAAA,cACtB;AAAA,cACAC,WAAU,SACN,yBAAyBC,OAAM,sCAAsCA,OAAM,sCAC3E;AAAA,cACJ;AAAA,YACF,EAAE,KAAK,IAAI;AACX,qBAAS,KAAK,YAAY,OAAO;AACjC,gBAAI,KAAK,EAAE;AAAA,UACb;AAAA,QACF,QAAQ;AAAA,QAAuB;AAAA,MACjC;AACA,aAAO;AAAA,IACT,GAAG,EAAE,WAAW,OAAO,OAAO,CAAC;AAE/B,eAAW,KAAK,GAAG,MAAM;AACzB,kBAAc,OAAO;AAAA,EACvB,QAAQ;AAAA,EAAkB;AAG1B,QAAM,aAAa,MAAM,KAAK,WAAW,EAAE,MAAM,OAAO,UAAU,MAAM,CAAC;AAGzE,MAAI,WAAW,SAAS,GAAG;AACzB,UAAM,KAAK,SAAS,CAAC,QAAQ;AAC3B,iBAAW,MAAM,IAAK,UAAS,eAAe,EAAE,GAAG,OAAO;AAAA,IAC5D,GAAG,UAAU,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC/B;AAEA,SAAO,EAAE,aAAa,WAAW;AACnC;",
+  "names": ["selectors", "style", "blurPx"]
+}

package/dist/seo-checker.d.ts

@@ -0,0 +1,33 @@
+/**
+ * SEO indexability checker — determines if a page is crawlable and indexable.
+ *
+ * Checks:
+ * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
+ * 2. <meta name="robots"> tag (noindex, nofollow)
+ * 3. Canonical URL (self-canonical vs cross-canonical)
+ * 4. rel="nofollow" on outbound links
+ * 5. hreflang tags (multilingual)
+ * 6. Open Graph completeness (og:title, og:description, og:image)
+ * 7. Structured data presence (JSON-LD)
+ * 8. Meta description
+ * 9. Title tag length (< 60 chars recommended)
+ * 10. URL structure (query params, fragments, excessive depth)
+ */
+import type { Page } from 'playwright';
+export interface SeoIssue {
+    rule: string;
+    severity: 'critical' | 'warning' | 'info';
+    message: string;
+    value?: string;
+}
+export interface SeoCheckResult {
+    url: string;
+    isIndexable: boolean;
+    canonicalUrl?: string;
+    isSelfCanonical?: boolean;
+    hreflang: Record<string, string>;
+    issues: SeoIssue[];
+    score: number;
+    checkedAt: string;
+}
+export declare function checkSeo(page: Page, url: string, responseHeaders?: Record<string, string>): Promise<SeoCheckResult>;

package/dist/seo-checker.js

@@ -0,0 +1,91 @@
+async function checkSeo(page, url, responseHeaders = {}) {
+  const issues = [];
+  const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
+  let isIndexable = true;
+  const robotsHeader = responseHeaders["x-robots-tag"] ?? "";
+  if (/noindex/i.test(robotsHeader)) {
+    isIndexable = false;
+    issues.push({ rule: "x-robots-noindex", severity: "critical", message: "X-Robots-Tag: noindex found in response header", value: robotsHeader });
+  }
+  const pageData = await page.evaluate(() => {
+    const meta = (name) => document.querySelector(`meta[name="${name}"]`)?.getAttribute("content") ?? "";
+    const og = (prop) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute("content") ?? "";
+    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute("href") ?? "";
+    const hreflang = {};
+    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
+      hreflang[el.getAttribute("hreflang")] = el.getAttribute("href") ?? "";
+    });
+    const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s) => s.textContent ?? "");
+    return {
+      robotsMeta: meta("robots"),
+      description: meta("description"),
+      title: document.title ?? "",
+      canonical,
+      hreflang,
+      ogTitle: og("title"),
+      ogDescription: og("description"),
+      ogImage: og("image"),
+      hasJsonLd: jsonLd.length > 0,
+      jsonLd
+    };
+  }).catch(() => ({
+    robotsMeta: "",
+    description: "",
+    title: "",
+    canonical: "",
+    hreflang: {},
+    ogTitle: "",
+    ogDescription: "",
+    ogImage: "",
+    hasJsonLd: false,
+    jsonLd: []
+  }));
+  if (/noindex/i.test(pageData.robotsMeta)) {
+    isIndexable = false;
+    issues.push({ rule: "meta-robots-noindex", severity: "critical", message: "Meta robots noindex found", value: pageData.robotsMeta });
+  }
+  let isSelfCanonical;
+  if (pageData.canonical) {
+    isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, "") === url.replace(/\/$/, "");
+    if (!isSelfCanonical) {
+      issues.push({ rule: "cross-canonical", severity: "warning", message: "Cross-canonical points elsewhere", value: pageData.canonical });
+    }
+  } else {
+    issues.push({ rule: "missing-canonical", severity: "warning", message: "No canonical link tag" });
+  }
+  if (!pageData.title) {
+    issues.push({ rule: "missing-title", severity: "critical", message: "Page has no title tag" });
+  } else if (pageData.title.length > 60) {
+    issues.push({ rule: "title-too-long", severity: "warning", message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
+  }
+  if (!pageData.description) {
+    issues.push({ rule: "missing-description", severity: "warning", message: "Missing meta description" });
+  } else if (pageData.description.length > 160) {
+    issues.push({ rule: "description-too-long", severity: "info", message: `Description ${pageData.description.length} chars (>160)` });
+  }
+  if (!pageData.ogTitle) issues.push({ rule: "missing-og-title", severity: "info", message: "Missing og:title" });
+  if (!pageData.ogImage) issues.push({ rule: "missing-og-image", severity: "info", message: "Missing og:image" });
+  if (!pageData.hasJsonLd) issues.push({ rule: "no-structured-data", severity: "info", message: "No JSON-LD structured data found" });
+  try {
+    const depth = new URL(url).pathname.split("/").filter(Boolean).length;
+    if (depth > 5) issues.push({ rule: "deep-url", severity: "info", message: `URL depth ${depth} (>5 levels)`, value: url });
+  } catch {
+  }
+  const criticals = issues.filter((i) => i.severity === "critical").length;
+  const warnings = issues.filter((i) => i.severity === "warning").length;
+  const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter((i) => i.severity === "info").length * 2);
+  return {
+    url,
+    isIndexable,
+    canonicalUrl: pageData.canonical || void 0,
+    isSelfCanonical,
+    hreflang: pageData.hreflang,
+    issues,
+    score,
+    checkedAt
+  };
+}
+export {
+  checkSeo
+};
+//# sourceMappingURL=seo-checker.js.map

package/dist/seo-checker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/seo-checker.ts"],
+  "sourcesContent": ["/**\n * SEO indexability checker \u2014 determines if a page is crawlable and indexable.\n *\n * Checks:\n * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)\n * 2. <meta name=\"robots\"> tag (noindex, nofollow)\n * 3. Canonical URL (self-canonical vs cross-canonical)\n * 4. rel=\"nofollow\" on outbound links\n * 5. hreflang tags (multilingual)\n * 6. Open Graph completeness (og:title, og:description, og:image)\n * 7. Structured data presence (JSON-LD)\n * 8. Meta description\n * 9. Title tag length (< 60 chars recommended)\n * 10. URL structure (query params, fragments, excessive depth)\n */\n\nimport type { Page } from 'playwright';\n\nexport interface SeoIssue {\n  rule: string;\n  severity: 'critical' | 'warning' | 'info';\n  message: string;\n  value?: string;\n}\n\nexport interface SeoCheckResult {\n  url: string;\n  isIndexable: boolean;\n  canonicalUrl?: string;\n  isSelfCanonical?: boolean;\n  hreflang: Record<string, string>; // lang \u2192 url\n  issues: SeoIssue[];\n  score: number; // 0-100\n  checkedAt: string;\n}\n\nexport async function checkSeo(page: Page, url: string, responseHeaders: Record<string, string> = {}): Promise<SeoCheckResult> {\n  const issues: SeoIssue[] = [];\n  const checkedAt = new Date().toISOString();\n  let isIndexable = true;\n\n  // 1. X-Robots-Tag\n  const robotsHeader = responseHeaders['x-robots-tag'] ?? '';\n  if (/noindex/i.test(robotsHeader)) {\n    isIndexable = false;\n    issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader });\n  }\n\n  // 2-9. Page-level checks\n  const pageData = await page.evaluate(() => {\n    const meta = (name: string) => document.querySelector(`meta[name=\"${name}\"]`)?.getAttribute('content') ?? '';\n    const og = (prop: string) => document.querySelector(`meta[property=\"og:${prop}\"]`)?.getAttribute('content') ?? '';\n    const canonical = document.querySelector('link[rel=\"canonical\"]')?.getAttribute('href') ?? '';\n    const hreflang: Record<string, string> = {};\n    document.querySelectorAll('link[rel=\"alternate\"][hreflang]').forEach((el: any) => {\n      hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? '';\n    });\n    const jsonLd = Array.from(document.querySelectorAll('script[type=\"application/ld+json\"]')).map((s: any) => s.textContent ?? '');\n    return {\n      robotsMeta: meta('robots'),\n      description: meta('description'),\n      title: document.title ?? '',\n      canonical,\n      hreflang,\n      ogTitle: og('title'),\n      ogDescription: og('description'),\n      ogImage: og('image'),\n      hasJsonLd: jsonLd.length > 0,\n      jsonLd,\n    };\n  }).catch(() => ({\n    robotsMeta: '', description: '', title: '', canonical: '', hreflang: {},\n    ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [],\n  }));\n\n  // 2. Meta robots\n  if (/noindex/i.test(pageData.robotsMeta)) {\n    isIndexable = false;\n    issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta });\n  }\n\n  // 3. Canonical\n  let isSelfCanonical: boolean | undefined;\n  if (pageData.canonical) {\n    isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\\/$/, '') === url.replace(/\\/$/, '');\n    if (!isSelfCanonical) {\n      issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical });\n    }\n  } else {\n    issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' });\n  }\n\n  // 4. Title\n  if (!pageData.title) {\n    issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' });\n  } else if (pageData.title.length > 60) {\n    issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });\n  }\n\n  // 5. Meta description\n  if (!pageData.description) {\n    issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' });\n  } else if (pageData.description.length > 160) {\n    issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` });\n  }\n\n  // 6. OG tags\n  if (!pageData.ogTitle) issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' });\n  if (!pageData.ogImage) issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' });\n\n  // 7. Structured data\n  if (!pageData.hasJsonLd) issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' });\n\n  // 8. URL depth\n  try {\n    const depth = new URL(url).pathname.split('/').filter(Boolean).length;\n    if (depth > 5) issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url });\n  } catch {}\n\n  const criticals = issues.filter(i => i.severity === 'critical').length;\n  const warnings = issues.filter(i => i.severity === 'warning').length;\n  const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2);\n\n  return {\n    url,\n    isIndexable,\n    canonicalUrl: pageData.canonical || undefined,\n    isSelfCanonical,\n    hreflang: pageData.hreflang,\n    issues,\n    score,\n    checkedAt,\n  };\n}\n"],
+  "mappings": "AAoCA,eAAsB,SAAS,MAAY,KAAa,kBAA0C,CAAC,GAA4B;AAC7H,QAAM,SAAqB,CAAC;AAC5B,QAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AACzC,MAAI,cAAc;AAGlB,QAAM,eAAe,gBAAgB,cAAc,KAAK;AACxD,MAAI,WAAW,KAAK,YAAY,GAAG;AACjC,kBAAc;AACd,WAAO,KAAK,EAAE,MAAM,oBAAoB,UAAU,YAAY,SAAS,kDAAkD,OAAO,aAAa,CAAC;AAAA,EAChJ;AAGA,QAAM,WAAW,MAAM,KAAK,SAAS,MAAM;AACzC,UAAM,OAAO,CAAC,SAAiB,SAAS,cAAc,cAAc,IAAI,IAAI,GAAG,aAAa,SAAS,KAAK;AAC1G,UAAM,KAAK,CAAC,SAAiB,SAAS,cAAc,qBAAqB,IAAI,IAAI,GAAG,aAAa,SAAS,KAAK;AAC/G,UAAM,YAAY,SAAS,cAAc,uBAAuB,GAAG,aAAa,MAAM,KAAK;AAC3F,UAAM,WAAmC,CAAC;AAC1C,aAAS,iBAAiB,iCAAiC,EAAE,QAAQ,CAAC,OAAY;AAChF,eAAS,GAAG,aAAa,UAAU,CAAC,IAAI,GAAG,aAAa,MAAM,KAAK;AAAA,IACrE,CAAC;AACD,UAAM,SAAS,MAAM,KAAK,SAAS,iBAAiB,oCAAoC,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,eAAe,EAAE;AAC9H,WAAO;AAAA,MACL,YAAY,KAAK,QAAQ;AAAA,MACzB,aAAa,KAAK,aAAa;AAAA,MAC/B,OAAO,SAAS,SAAS;AAAA,MACzB;AAAA,MACA;AAAA,MACA,SAAS,GAAG,OAAO;AAAA,MACnB,eAAe,GAAG,aAAa;AAAA,MAC/B,SAAS,GAAG,OAAO;AAAA,MACnB,WAAW,OAAO,SAAS;AAAA,MAC3B;AAAA,IACF;AAAA,EACF,CAAC,EAAE,MAAM,OAAO;AAAA,IACd,YAAY;AAAA,IAAI,aAAa;AAAA,IAAI,OAAO;AAAA,IAAI,WAAW;AAAA,IAAI,UAAU,CAAC;AAAA,IACtE,SAAS;AAAA,IAAI,eAAe;AAAA,IAAI,SAAS;AAAA,IAAI,WAAW;AAAA,IAAO,QAAQ,CAAC;AAAA,EAC1E,EAAE;AAGF,MAAI,WAAW,KAAK,SAAS,UAAU,GAAG;AACxC,kBAAc;AACd,WAAO,KAAK,EAAE,MAAM,uBAAuB,UAAU,YAAY,SAAS,6BAA6B,OAAO,SAAS,WAAW,CAAC;AAAA,EACrI;AAGA,MAAI;AACJ,MAAI,SAAS,WAAW;AACtB,sBAAkB,SAAS,cAAc,OAAO,SAAS,UAAU,QAAQ,OAAO,EAAE,MAAM,IAAI,QAAQ,OAAO,EAAE;AAC/G,QAAI,CAAC,iBAAiB;AACpB,aAAO,KAAK,EAAE,MAAM,mBAAmB,UAAU,WAAW,SAAS,oCAAoC,OAAO,SAAS,UAAU,CAAC;AAAA,IACtI;AAAA,EACF,OAAO;AACL,WAAO,KAAK,EAAE,MAAM,qBAAqB,UAAU,WAAW,SAAS,wBAAwB,CAAC;AAAA,EAClG;AAGA,MAAI,CAAC,SAAS,OAAO;AACnB,WAAO,KAAK,EAAE,MAAM,iBAAiB,UAAU,YAAY,SAAS,wBAAwB,CAAC;AAAA,EAC/F,WAAW,SAAS,MAAM,SAAS,IAAI;AACrC,WAAO,KAAK,EAAE,MAAM,kBAAkB,UAAU,WAAW,SAAS,SAAS,SAAS,MAAM,MAAM,gBAAgB,OAAO,SAAS,MAAM,CAAC;AAAA,EAC3I;AAGA,MAAI,CAAC,SAAS,aAAa;AACzB,WAAO,KAAK,EAAE,MAAM,uBAAuB,UAAU,WAAW,SAAS,2BAA2B,CAAC;AAAA,EACvG,WAAW,SAAS,YAAY,SAAS,KAAK;AAC5C,WAAO,KAAK,EAAE,MAAM,wBAAwB,UAAU,QAAQ,SAAS,eAAe,SAAS,YAAY,MAAM,gBAAgB,CAAC;AAAA,EACpI;AAGA,MAAI,CAAC,SAAS,QAAS,QAAO,KAAK,EAAE,MAAM,oBAAoB,UAAU,QAAQ,SAAS,mBAAmB,CAAC;AAC9G,MAAI,CAAC,SAAS,QAAS,QAAO,KAAK,EAAE,MAAM,oBAAoB,UAAU,QAAQ,SAAS,mBAAmB,CAAC;AAG9G,MAAI,CAAC,SAAS,UAAW,QAAO,KAAK,EAAE,MAAM,sBAAsB,UAAU,QAAQ,SAAS,mCAAmC,CAAC;AAGlI,MAAI;AACF,UAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO,EAAE;AAC/D,QAAI,QAAQ,EAAG,QAAO,KAAK,EAAE,MAAM,YAAY,UAAU,QAAQ,SAAS,aAAa,KAAK,gBAAgB,OAAO,IAAI,CAAC;AAAA,EAC1H,QAAQ;AAAA,EAAC;AAET,QAAM,YAAY,OAAO,OAAO,OAAK,EAAE,aAAa,UAAU,EAAE;AAChE,QAAM,WAAW,OAAO,OAAO,OAAK,EAAE,aAAa,SAAS,EAAE;AAC9D,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,KAAK,WAAW,IAAI,OAAO,OAAO,OAAK,EAAE,aAAa,MAAM,EAAE,SAAS,CAAC;AAEpH,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,SAAS,aAAa;AAAA,IACpC;AAAA,IACA,UAAU,SAAS;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/service-worker-interceptor.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Service Worker detection — inspects SW registrations, cache storage, background
+ * sync queues, and PWA installability for a crawled page.
+ * Surfaces offline-first contracts and push-capable surfaces that HTTP-only
+ * crawlers cannot observe.
+ */
+export interface SwRegistration {
+    scope: string;
+    scriptURL: string | null;
+    state: 'active' | 'installing' | 'waiting' | 'none';
+    updateViaCache: string;
+}
+export interface SwCacheEntry {
+    cacheName: string;
+    entryCount: number;
+    sampleUrls: string[];
+}
+export interface BackgroundSyncInfo {
+    scope: string;
+    pendingTags: string[];
+}
+export interface PwaInfo {
+    isInstallable: boolean;
+    manifestUrl: string | null;
+    hasPushPermission: boolean;
+}
+export interface ServiceWorkerResult {
+    supported: boolean;
+    active: boolean;
+    registrations: SwRegistration[];
+    cacheStorages: SwCacheEntry[];
+    backgroundSync: BackgroundSyncInfo[];
+    pwa: PwaInfo;
+    totalCachedEntries: number;
+}
+export declare function inspectServiceWorker(page: any): Promise<ServiceWorkerResult>;

package/dist/service-worker-interceptor.js

@@ -0,0 +1,136 @@
+const EMPTY_RESULT = {
+  supported: false,
+  active: false,
+  registrations: [],
+  cacheStorages: [],
+  backgroundSync: [],
+  pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },
+  totalCachedEntries: 0
+};
+async function detectServiceWorkers(page) {
+  const result = await page.evaluate(async () => {
+    if (!("serviceWorker" in navigator)) return { supported: false, active: false, registrations: [] };
+    try {
+      const controller = navigator.serviceWorker.controller;
+      const regs = await navigator.serviceWorker.getRegistrations();
+      return {
+        supported: true,
+        active: !!controller,
+        registrations: regs.map((r) => ({
+          scope: r.scope,
+          scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,
+          state: r.active ? "active" : r.installing ? "installing" : r.waiting ? "waiting" : "none",
+          updateViaCache: r.updateViaCache
+        }))
+      };
+    } catch {
+      return { supported: true, active: false, registrations: [] };
+    }
+  });
+  return result;
+}
+async function enumerateCacheStorage(page) {
+  const caches = await page.evaluate(async () => {
+    if (!("caches" in window)) return [];
+    try {
+      const names = await window.caches.keys();
+      const result = [];
+      for (const name of names) {
+        try {
+          const cache = await window.caches.open(name);
+          const keys = await cache.keys();
+          result.push({
+            cacheName: name,
+            entryCount: keys.length,
+            sampleUrls: keys.slice(0, 15).map((k) => {
+              try {
+                return new URL(k.url).pathname;
+              } catch {
+                return k.url.slice(0, 80);
+              }
+            })
+          });
+        } catch {
+        }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return caches;
+}
+async function detectBackgroundSync(page) {
+  const syncInfo = await page.evaluate(async () => {
+    if (!("serviceWorker" in navigator)) return [];
+    try {
+      const regs = await navigator.serviceWorker.getRegistrations();
+      const result = [];
+      for (const reg of regs) {
+        if ("sync" in reg) {
+          try {
+            const tags = await reg.sync.getTags();
+            result.push({ scope: reg.scope, pendingTags: tags || [] });
+          } catch {
+            result.push({ scope: reg.scope, pendingTags: [] });
+          }
+        }
+      }
+      return result;
+    } catch {
+      return [];
+    }
+  });
+  return syncInfo;
+}
+async function detectPwa(page) {
+  const pwa = await page.evaluate(() => {
+    const manifestEl = document.querySelector('link[rel="manifest"]');
+    const manifestUrl = manifestEl?.getAttribute("href") || null;
+    const hasSW = "serviceWorker" in navigator && !!navigator.serviceWorker.controller;
+    let hasPushPermission = false;
+    try {
+      hasPushPermission = Notification.permission === "granted";
+    } catch {
+    }
+    return {
+      isInstallable: !!(manifestUrl && hasSW),
+      manifestUrl,
+      hasPushPermission
+    };
+  });
+  return pwa;
+}
+async function inspectServiceWorker(page) {
+  try {
+    const url = page.url();
+    const isSecureContext = url.startsWith("https://") || /^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?/.test(url);
+    if (!isSecureContext) return { ...EMPTY_RESULT };
+    const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([
+      detectServiceWorkers(page),
+      enumerateCacheStorage(page),
+      detectBackgroundSync(page),
+      detectPwa(page)
+    ]);
+    const sw = swResult.status === "fulfilled" ? swResult.value : { supported: false, active: false, registrations: [] };
+    const cacheStorages = cacheResult.status === "fulfilled" ? cacheResult.value : [];
+    const backgroundSync = syncResult.status === "fulfilled" ? syncResult.value : [];
+    const pwa = pwaResult.status === "fulfilled" ? pwaResult.value : EMPTY_RESULT.pwa;
+    const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);
+    return {
+      supported: sw.supported,
+      active: sw.active,
+      registrations: sw.registrations,
+      cacheStorages,
+      backgroundSync,
+      pwa,
+      totalCachedEntries
+    };
+  } catch {
+    return { ...EMPTY_RESULT };
+  }
+}
+export {
+  inspectServiceWorker
+};
+//# sourceMappingURL=service-worker-interceptor.js.map

package/dist/service-worker-interceptor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/service-worker-interceptor.ts"],
+  "sourcesContent": ["/**\n * Service Worker detection \u2014 inspects SW registrations, cache storage, background\n * sync queues, and PWA installability for a crawled page.\n * Surfaces offline-first contracts and push-capable surfaces that HTTP-only\n * crawlers cannot observe.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface SwRegistration {\n  scope: string;\n  scriptURL: string | null;\n  state: 'active' | 'installing' | 'waiting' | 'none';\n  updateViaCache: string;\n}\n\nexport interface SwCacheEntry {\n  cacheName: string;\n  entryCount: number;\n  sampleUrls: string[];\n}\n\nexport interface BackgroundSyncInfo {\n  scope: string;\n  pendingTags: string[];\n}\n\nexport interface PwaInfo {\n  isInstallable: boolean;\n  manifestUrl: string | null;\n  hasPushPermission: boolean;\n}\n\nexport interface ServiceWorkerResult {\n  supported: boolean;\n  active: boolean;\n  registrations: SwRegistration[];\n  cacheStorages: SwCacheEntry[];\n  backgroundSync: BackgroundSyncInfo[];\n  pwa: PwaInfo;\n  totalCachedEntries: number;\n}\n\nconst EMPTY_RESULT: ServiceWorkerResult = {\n  supported: false,\n  active: false,\n  registrations: [],\n  cacheStorages: [],\n  backgroundSync: [],\n  pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },\n  totalCachedEntries: 0,\n};\n\nasync function detectServiceWorkers(\n  page: any,\n): Promise<{ supported: boolean; active: boolean; registrations: SwRegistration[] }> {\n  const result = await page.evaluate(async () => {\n    if (!('serviceWorker' in navigator)) return { supported: false, active: false, registrations: [] };\n    try {\n      const controller = navigator.serviceWorker.controller;\n      const regs = await navigator.serviceWorker.getRegistrations();\n      return {\n        supported: true,\n        active: !!controller,\n        registrations: regs.map(r => ({\n          scope: r.scope,\n          scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,\n          state: r.active ? 'active' : r.installing ? 'installing' : r.waiting ? 'waiting' : 'none',\n          updateViaCache: r.updateViaCache,\n        })),\n      };\n    } catch {\n      return { supported: true, active: false, registrations: [] };\n    }\n  });\n  return result as { supported: boolean; active: boolean; registrations: SwRegistration[] };\n}\n\nasync function enumerateCacheStorage(page: any): Promise<SwCacheEntry[]> {\n  const caches = await page.evaluate(async () => {\n    if (!('caches' in window)) return [];\n    try {\n      const names = await window.caches.keys();\n      const result = [];\n      for (const name of names) {\n        try {\n          const cache = await window.caches.open(name);\n          const keys = await cache.keys();\n          result.push({\n            cacheName: name,\n            entryCount: keys.length,\n            sampleUrls: keys.slice(0, 15).map(k => {\n              try { return new URL(k.url).pathname; } catch { return k.url.slice(0, 80); }\n            }),\n          });\n        } catch { /* skip inaccessible cache */ }\n      }\n      return result;\n    } catch {\n      return [];\n    }\n  });\n  return caches as SwCacheEntry[];\n}\n\nasync function detectBackgroundSync(page: any): Promise<BackgroundSyncInfo[]> {\n  const syncInfo = await page.evaluate(async () => {\n    if (!('serviceWorker' in navigator)) return [];\n    try {\n      const regs = await navigator.serviceWorker.getRegistrations();\n      const result = [];\n      for (const reg of regs) {\n        if ('sync' in reg) {\n          try {\n            const tags = await (reg as any).sync.getTags();\n            result.push({ scope: reg.scope, pendingTags: tags || [] });\n          } catch {\n            result.push({ scope: reg.scope, pendingTags: [] });\n          }\n        }\n      }\n      return result;\n    } catch {\n      return [];\n    }\n  });\n  return syncInfo as BackgroundSyncInfo[];\n}\n\nasync function detectPwa(page: any): Promise<PwaInfo> {\n  const pwa = await page.evaluate(() => {\n    const manifestEl = document.querySelector('link[rel=\"manifest\"]');\n    const manifestUrl = manifestEl?.getAttribute('href') || null;\n    const hasSW = 'serviceWorker' in navigator && !!navigator.serviceWorker.controller;\n    let hasPushPermission = false;\n    try { hasPushPermission = Notification.permission === 'granted'; } catch { /* ignore */ }\n    return {\n      isInstallable: !!(manifestUrl && hasSW),\n      manifestUrl,\n      hasPushPermission,\n    };\n  });\n  return pwa as PwaInfo;\n}\n\nexport async function inspectServiceWorker(page: any): Promise<ServiceWorkerResult> {\n  try {\n    // Allow SW on HTTPS or localhost (http://localhost and http://127.0.0.1 support SW)\n    const url = page.url();\n    const isSecureContext = url.startsWith('https://') ||\n      /^http:\\/\\/(localhost|127\\.0\\.0\\.1)(:\\d+)?/.test(url);\n    if (!isSecureContext) return { ...EMPTY_RESULT };\n\n    const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([\n      detectServiceWorkers(page),\n      enumerateCacheStorage(page),\n      detectBackgroundSync(page),\n      detectPwa(page),\n    ]);\n\n    const sw = swResult.status === 'fulfilled'\n      ? swResult.value\n      : { supported: false, active: false, registrations: [] };\n\n    const cacheStorages = cacheResult.status === 'fulfilled' ? cacheResult.value : [];\n    const backgroundSync = syncResult.status === 'fulfilled' ? syncResult.value : [];\n    const pwa = pwaResult.status === 'fulfilled'\n      ? pwaResult.value\n      : EMPTY_RESULT.pwa;\n\n    const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);\n\n    return {\n      supported: sw.supported,\n      active: sw.active,\n      registrations: sw.registrations,\n      cacheStorages,\n      backgroundSync,\n      pwa,\n      totalCachedEntries,\n    };\n  } catch {\n    return { ...EMPTY_RESULT };\n  }\n}\n"],
+  "mappings": "AA2CA,MAAM,eAAoC;AAAA,EACxC,WAAW;AAAA,EACX,QAAQ;AAAA,EACR,eAAe,CAAC;AAAA,EAChB,eAAe,CAAC;AAAA,EAChB,gBAAgB,CAAC;AAAA,EACjB,KAAK,EAAE,eAAe,OAAO,aAAa,MAAM,mBAAmB,MAAM;AAAA,EACzE,oBAAoB;AACtB;AAEA,eAAe,qBACb,MACmF;AACnF,QAAM,SAAS,MAAM,KAAK,SAAS,YAAY;AAC7C,QAAI,EAAE,mBAAmB,WAAY,QAAO,EAAE,WAAW,OAAO,QAAQ,OAAO,eAAe,CAAC,EAAE;AACjG,QAAI;AACF,YAAM,aAAa,UAAU,cAAc;AAC3C,YAAM,OAAO,MAAM,UAAU,cAAc,iBAAiB;AAC5D,aAAO;AAAA,QACL,WAAW;AAAA,QACX,QAAQ,CAAC,CAAC;AAAA,QACV,eAAe,KAAK,IAAI,QAAM;AAAA,UAC5B,OAAO,EAAE;AAAA,UACT,WAAW,EAAE,QAAQ,aAAa,EAAE,YAAY,aAAa,EAAE,SAAS,aAAa;AAAA,UACrF,OAAO,EAAE,SAAS,WAAW,EAAE,aAAa,eAAe,EAAE,UAAU,YAAY;AAAA,UACnF,gBAAgB,EAAE;AAAA,QACpB,EAAE;AAAA,MACJ;AAAA,IACF,QAAQ;AACN,aAAO,EAAE,WAAW,MAAM,QAAQ,OAAO,eAAe,CAAC,EAAE;AAAA,IAC7D;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,sBAAsB,MAAoC;AACvE,QAAM,SAAS,MAAM,KAAK,SAAS,YAAY;AAC7C,QAAI,EAAE,YAAY,QAAS,QAAO,CAAC;AACnC,QAAI;AACF,YAAM,QAAQ,MAAM,OAAO,OAAO,KAAK;AACvC,YAAM,SAAS,CAAC;AAChB,iBAAW,QAAQ,OAAO;AACxB,YAAI;AACF,gBAAM,QAAQ,MAAM,OAAO,OAAO,KAAK,IAAI;AAC3C,gBAAM,OAAO,MAAM,MAAM,KAAK;AAC9B,iBAAO,KAAK;AAAA,YACV,WAAW;AAAA,YACX,YAAY,KAAK;AAAA,YACjB,YAAY,KAAK,MAAM,GAAG,EAAE,EAAE,IAAI,OAAK;AACrC,kBAAI;AAAE,uBAAO,IAAI,IAAI,EAAE,GAAG,EAAE;AAAA,cAAU,QAAQ;AAAE,uBAAO,EAAE,IAAI,MAAM,GAAG,EAAE;AAAA,cAAG;AAAA,YAC7E,CAAC;AAAA,UACH,CAAC;AAAA,QACH,QAAQ;AAAA,QAAgC;AAAA,MAC1C;AACA,aAAO;AAAA,IACT,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,qBAAqB,MAA0C;AAC5E,QAAM,WAAW,MAAM,KAAK,SAAS,YAAY;AAC/C,QAAI,EAAE,mBAAmB,WAAY,QAAO,CAAC;AAC7C,QAAI;AACF,YAAM,OAAO,MAAM,UAAU,cAAc,iBAAiB;AAC5D,YAAM,SAAS,CAAC;AAChB,iBAAW,OAAO,MAAM;AACtB,YAAI,UAAU,KAAK;AACjB,cAAI;AACF,kBAAM,OAAO,MAAO,IAAY,KAAK,QAAQ;AAC7C,mBAAO,KAAK,EAAE,OAAO,IAAI,OAAO,aAAa,QAAQ,CAAC,EAAE,CAAC;AAAA,UAC3D,QAAQ;AACN,mBAAO,KAAK,EAAE,OAAO,IAAI,OAAO,aAAa,CAAC,EAAE,CAAC;AAAA,UACnD;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,UAAU,MAA6B;AACpD,QAAM,MAAM,MAAM,KAAK,SAAS,MAAM;AACpC,UAAM,aAAa,SAAS,cAAc,sBAAsB;AAChE,UAAM,cAAc,YAAY,aAAa,MAAM,KAAK;AACxD,UAAM,QAAQ,mBAAmB,aAAa,CAAC,CAAC,UAAU,cAAc;AACxE,QAAI,oBAAoB;AACxB,QAAI;AAAE,0BAAoB,aAAa,eAAe;AAAA,IAAW,QAAQ;AAAA,IAAe;AACxF,WAAO;AAAA,MACL,eAAe,CAAC,EAAE,eAAe;AAAA,MACjC;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAsB,qBAAqB,MAAyC;AAClF,MAAI;AAEF,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,kBAAkB,IAAI,WAAW,UAAU,KAC/C,4CAA4C,KAAK,GAAG;AACtD,QAAI,CAAC,gBAAiB,QAAO,EAAE,GAAG,aAAa;AAE/C,UAAM,CAAC,UAAU,aAAa,YAAY,SAAS,IAAI,MAAM,QAAQ,WAAW;AAAA,MAC9E,qBAAqB,IAAI;AAAA,MACzB,sBAAsB,IAAI;AAAA,MAC1B,qBAAqB,IAAI;AAAA,MACzB,UAAU,IAAI;AAAA,IAChB,CAAC;AAED,UAAM,KAAK,SAAS,WAAW,cAC3B,SAAS,QACT,EAAE,WAAW,OAAO,QAAQ,OAAO,eAAe,CAAC,EAAE;AAEzD,UAAM,gBAAgB,YAAY,WAAW,cAAc,YAAY,QAAQ,CAAC;AAChF,UAAM,iBAAiB,WAAW,WAAW,cAAc,WAAW,QAAQ,CAAC;AAC/E,UAAM,MAAM,UAAU,WAAW,cAC7B,UAAU,QACV,aAAa;AAEjB,UAAM,qBAAqB,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC;AAEjF,WAAO;AAAA,MACL,WAAW,GAAG;AAAA,MACd,QAAQ,GAAG;AAAA,MACX,eAAe,GAAG;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,QAAQ;AACN,WAAO,EAAE,GAAG,aAAa;AAAA,EAC3B;AACF;",
+  "names": []
+}

package/dist/session-capture.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Opens a headed (visible) browser at appUrl and returns a captureId.
+ * The user logs in manually, then calls completeSessionCapture() to harvest storageState.
+ * Sessions expire after 10 minutes if not completed.
+ *
+ * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
+ * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
+ */
+export declare function startSessionCapture(appUrl: string): Promise<string>;
+/** Captures storageState from the open browser, closes it, and returns the JSON string. */
+export declare function completeSessionCapture(captureId: string): Promise<string>;
+/** Closes the browser without capturing — called on Cancel. */
+export declare function cancelSessionCapture(captureId: string): Promise<void>;
+/**
+ * Headless login + session capture — works on Docker/cloud (no display needed).
+ * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
+ * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
+ *
+ * Returns { storageState, cookies, localStorage } on success.
+ * Throws a human-readable error on failure.
+ */
+export declare function validateAndCaptureSession(appUrl: string, creds: {
+    username: string;
+    password: string;
+    totpSecret?: string;
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+}, llmConfig?: {
+    modelName: string;
+    apiKey: string;
+}, captchaOpts?: {
+    apiKey?: string;
+    provider?: '2captcha' | 'capsolver';
+}, loginInstructions?: string): Promise<{
+    storageState: string;
+    cookies: number;
+    localStorage: number;
+    finalUrl?: string;
+}>;