@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/locator-extractor.js

@@ -0,0 +1,220 @@
+function parseNotes(notes) {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === "string" ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === "object") return parsed;
+  } catch {
+  }
+  return {};
+}
+function mapRole(role) {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === "button" || r === "submit" || r === "reset") return "button";
+  if (r === "link" || r === "a" || r === "anchor") return "link";
+  if (r === "textbox" || r === "input" || r === "text" || r === "email" || r === "password" || r === "tel" || r === "number" || r === "search") return "textbox";
+  if (r === "combobox" || r === "select" || r === "dropdown") return "combobox";
+  if (r === "checkbox") return "checkbox";
+  if (r === "radio") return "radio";
+  if (r === "heading" || r === "h1" || r === "h2" || r === "h3" || r === "h4" || r === "h5" || r === "h6") return "heading";
+  if (r === "img" || r === "image") return "img";
+  if (r === "nav" || r === "navigation") return "navigation";
+  return r;
+}
+function toKey(text) {
+  return text.replace(/[^a-zA-Z0-9\s]/g, " ").replace(/\s+(.)/g, (_, c) => c.toUpperCase()).replace(/^(.)/, (_, c) => c.toLowerCase()).replace(/\s/g, "").slice(0, 40) || "element";
+}
+function elementToPlaywrightLocator(element) {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs["data-testid"];
+  const ariaLabel = attrs["aria-label"] || element.meaning;
+  const id = attrs["id"];
+  const placeholder = attrs["placeholder"];
+  const name = attrs["name"];
+  const text = (element.text ?? "").trim();
+  const meaning = (element.meaning ?? "").trim();
+  const role = mapRole(element.role);
+  const ariaState = {};
+  if (element.ariaState && typeof element.ariaState === "object") {
+    for (const [k, v] of Object.entries(element.ariaState)) {
+      if (typeof v === "string") ariaState[k] = v;
+    }
+  }
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || "element";
+  const key = toKey(label);
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? "element",
+      confidence: "high",
+      source: "testid",
+      description: meaning || testId
+    };
+  }
+  if (role === "textbox" || role === "combobox") {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "label",
+        description: ariaLabel
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "placeholder",
+        description: placeholder
+      };
+    }
+  }
+  if ((role === "checkbox" || role === "radio") && meaning) {
+    const isChecked = ariaState["checked"];
+    const stateOpt = isChecked === "true" ? `, { name: ${JSON.stringify(meaning)}, checked: true }` : isChecked === "false" ? `, { name: ${JSON.stringify(meaning)}, checked: false }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role,
+      confidence: "high",
+      source: "role-name",
+      description: meaning
+    };
+  }
+  if ((role === "checkbox" || role === "radio") && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: "high",
+      source: "label",
+      description: ariaLabel
+    };
+  }
+  const tabLikeRoles = ["tab", "option", "treeitem", "menuitem", "menuitemcheckbox", "menuitemradio", "gridcell"];
+  if (tabLikeRoles.includes(role ?? "") && meaning) {
+    const isSelected = ariaState["selected"];
+    const isCurrent = ariaState["current"];
+    const stateOpt = isSelected === "true" ? `, { name: ${JSON.stringify(meaning)}, selected: true }` : isCurrent ? `, { name: ${JSON.stringify(meaning)} }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role ?? "element",
+      confidence: "high",
+      source: "role-name",
+      description: meaning
+    };
+  }
+  if ((role === "button" || role === "link") && meaning) {
+    const isPressed = ariaState["pressed"];
+    const stateOpt = isPressed === "true" ? `, { name: ${JSON.stringify(meaning)}, pressed: true }` : isPressed === "false" ? `, { name: ${JSON.stringify(meaning)}, pressed: false }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role,
+      confidence: "high",
+      source: "role-name",
+      description: meaning
+    };
+  }
+  if (role === "heading" && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: "heading",
+      confidence: "medium",
+      source: "heading",
+      description: name2
+    };
+  }
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || text || element.selector
+    };
+  }
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify("#" + id)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || id
+    };
+  }
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? "text",
+      confidence: "low",
+      source: "text",
+      description: t
+    };
+  }
+  return null;
+}
+const META_ANNOTATION_RE = /^(css animations|scroll effects detected|design tokens?\s*:|font imports?:|contrast ratio|aria-label audit|landmark|wcag|color contrast|background.*foreground.*card|--[\w-]+,\s*--)/i;
+const AUTH_WALL_TEXT_RE = /sign\s*in|log\s*in|platform access required|please sign in|access required|authentication required|session expired|unauthorized/i;
+function isMetaAnnotation(el) {
+  const t = (el.meaning ?? el.text ?? "").trim();
+  if (!t) return false;
+  if (META_ANNOTATION_RE.test(t)) return true;
+  const interactiveRoles = /* @__PURE__ */ new Set(["button", "link", "textbox", "combobox", "checkbox", "radio", "tab", "option", "menuitem"]);
+  if (t.length > 120 && !interactiveRoles.has((el.role ?? "").toLowerCase())) return true;
+  if ((t.match(/--[\w-]+/g) ?? []).length > 3) return true;
+  return false;
+}
+function elementsToPlaywrightLocators(elements) {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = /* @__PURE__ */ new Map();
+  for (const el of elements) {
+    if (isMetaAnnotation(el)) continue;
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+  return Array.from(byKey.values());
+}
+function detectAuthWallLocators(locators) {
+  if (locators.length === 0) return null;
+  const lowCount = locators.filter((l) => l.confidence === "low").length;
+  const hasAuthText = locators.some((l) => AUTH_WALL_TEXT_RE.test(l.description));
+  const highCount = locators.filter((l) => l.confidence === "high").length;
+  if (hasAuthText && highCount === 0 && lowCount / locators.length > 0.5) {
+    return "These locators appear to be from a login/access wall, not the target page. The page requires authentication. Add credentials in Project Settings and re-crawl to get accurate locators for the protected content.";
+  }
+  return null;
+}
+function formatLocatorsForPrompt(locators) {
+  if (locators.length === 0) return "";
+  const lines = locators.map(
+    (l) => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    "EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):",
+    ...lines
+  ].join("\n");
+}
+export {
+  detectAuthWallLocators,
+  elementToPlaywrightLocator,
+  elementsToPlaywrightLocators,
+  formatLocatorsForPrompt
+};
+//# sourceMappingURL=locator-extractor.js.map

package/dist/locator-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/locator-extractor.ts"],
+  "sourcesContent": ["/**\n * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.\n * Priority: data-testid \u2192 aria-label/placeholder (inputs) \u2192 role+name (buttons/links) \u2192 heading \u2192 text \u2192 id\n */\n\nexport interface PlaywrightLocator {\n  key: string;           // camelCase property name for page object\n  expression: string;    // e.g. \"page.getByRole('button', { name: 'Submit' })\"\n  elementType: string;   // button | input | link | heading | text | element\n  confidence: 'high' | 'medium' | 'low';\n  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';\n  description: string;   // human-readable label\n}\n\n/** Parse the notes JSON string stored on Element records. */\nfunction parseNotes(notes: unknown): Record<string, string> {\n  if (!notes) return {};\n  try {\n    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;\n    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;\n  } catch {}\n  return {};\n}\n\n/** Map crawler role strings to Playwright role values. */\nfunction mapRole(role?: string | null): string | null {\n  if (!role) return null;\n  const r = role.toLowerCase().trim();\n  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';\n  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';\n  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';\n  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';\n  if (r === 'checkbox') return 'checkbox';\n  if (r === 'radio') return 'radio';\n  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';\n  if (r === 'img' || r === 'image') return 'img';\n  if (r === 'nav' || r === 'navigation') return 'navigation';\n  return r;\n}\n\n/** Convert \"Submit Form Button\" \u2192 \"submitFormButton\" */\nfunction toKey(text: string): string {\n  return text\n    .replace(/[^a-zA-Z0-9\\s]/g, ' ')\n    .replace(/\\s+(.)/g, (_, c) => c.toUpperCase())\n    .replace(/^(.)/, (_, c) => c.toLowerCase())\n    .replace(/\\s/g, '')\n    .slice(0, 40)\n    || 'element';\n}\n\n/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */\nexport function elementToPlaywrightLocator(element: {\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n  ariaState?: Record<string, unknown> | null;\n}): PlaywrightLocator | null {\n  const attrs = parseNotes(element.notes);\n  const testId = attrs['data-testid'];\n  const ariaLabel = attrs['aria-label'] || element.meaning;\n  const id = attrs['id'];\n  const placeholder = attrs['placeholder'];\n  const name = attrs['name'];\n  const text = (element.text ?? '').trim();\n  const meaning = (element.meaning ?? '').trim();\n  const role = mapRole(element.role);\n  const ariaState: Record<string, string> = {};\n  if (element.ariaState && typeof element.ariaState === 'object') {\n    for (const [k, v] of Object.entries(element.ariaState)) {\n      if (typeof v === 'string') ariaState[k] = v;\n    }\n  }\n\n  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';\n  const key = toKey(label);\n\n  // Priority 1: data-testid\n  if (testId) {\n    return {\n      key,\n      expression: `page.getByTestId(${JSON.stringify(testId)})`,\n      elementType: role ?? 'element',\n      confidence: 'high',\n      source: 'testid',\n      description: meaning || testId,\n    };\n  }\n\n  // Priority 2: input/combobox/textarea with label \u2192 getByLabel; with placeholder \u2192 getByPlaceholder\n  if (role === 'textbox' || role === 'combobox') {\n    if (ariaLabel) {\n      return {\n        key,\n        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'label',\n        description: ariaLabel,\n      };\n    }\n    if (placeholder) {\n      return {\n        key,\n        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'placeholder',\n        description: placeholder,\n      };\n    }\n  }\n\n  // Priority 3: checkbox/radio \u2192 getByRole with optional checked state from ariaState\n  if ((role === 'checkbox' || role === 'radio') && meaning) {\n    const isChecked = ariaState['checked'];\n    const stateOpt = isChecked === 'true' ? `, { name: ${JSON.stringify(meaning)}, checked: true }` : isChecked === 'false' ? `, { name: ${JSON.stringify(meaning)}, checked: false }` : `, { name: ${JSON.stringify(meaning)} }`;\n    return {\n      key,\n      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,\n      elementType: role,\n      confidence: 'high',\n      source: 'role-name',\n      description: meaning,\n    };\n  }\n  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {\n    return {\n      key,\n      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n      elementType: role,\n      confidence: 'high',\n      source: 'label',\n      description: ariaLabel,\n    };\n  }\n\n  // Priority 3b: tab/option/treeitem \u2192 getByRole with optional selected state\n  const tabLikeRoles = ['tab', 'option', 'treeitem', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'gridcell'];\n  if (tabLikeRoles.includes(role ?? '') && meaning) {\n    const isSelected = ariaState['selected'];\n    const isCurrent = ariaState['current'];\n    const stateOpt = isSelected === 'true' ? `, { name: ${JSON.stringify(meaning)}, selected: true }` : isCurrent ? `, { name: ${JSON.stringify(meaning)} }` : `, { name: ${JSON.stringify(meaning)} }`;\n    return {\n      key,\n      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,\n      elementType: role ?? 'element',\n      confidence: 'high',\n      source: 'role-name',\n      description: meaning,\n    };\n  }\n\n  // Priority 4: button/link with meaningful name \u2192 getByRole; include pressed state for toggles\n  if ((role === 'button' || role === 'link') && meaning) {\n    const isPressed = ariaState['pressed'];\n    const stateOpt = isPressed === 'true' ? `, { name: ${JSON.stringify(meaning)}, pressed: true }` : isPressed === 'false' ? `, { name: ${JSON.stringify(meaning)}, pressed: false }` : `, { name: ${JSON.stringify(meaning)} }`;\n    return {\n      key,\n      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,\n      elementType: role,\n      confidence: 'high',\n      source: 'role-name',\n      description: meaning,\n    };\n  }\n\n  // Priority 5: heading \u2192 getByRole heading\n  if (role === 'heading' && (meaning || text)) {\n    const name2 = meaning || text;\n    return {\n      key,\n      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,\n      elementType: 'heading',\n      confidence: 'medium',\n      source: 'heading',\n      description: name2,\n    };\n  }\n\n  // Priority 6: existing CSS selector from crawler\n  if (element.selector && element.selector.length > 0) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify(element.selector)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || text || element.selector,\n    };\n  }\n\n  // Priority 7: id from attrs\n  if (id) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify('#' + id)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || id,\n    };\n  }\n\n  // Priority 8: fallback to text content\n  if (meaning || text) {\n    const t = meaning || text;\n    return {\n      key,\n      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,\n      elementType: role ?? 'text',\n      confidence: 'low',\n      source: 'text',\n      description: t,\n    };\n  }\n\n  return null;\n}\n\n// Patterns that indicate crawl-analysis meta-text stored as element records (not real UI elements).\n// These come from accessibility-auditor, screen-reader analysis, or CSS inspector annotations.\nconst META_ANNOTATION_RE = /^(css animations|scroll effects detected|design tokens?\\s*:|font imports?:|contrast ratio|aria-label audit|landmark|wcag|color contrast|background.*foreground.*card|--[\\w-]+,\\s*--)/i;\n\n// Auth-wall element patterns \u2014 if >50% of locators match these, the page was an auth wall.\nconst AUTH_WALL_TEXT_RE = /sign\\s*in|log\\s*in|platform access required|please sign in|access required|authentication required|session expired|unauthorized/i;\n\n/**\n * Returns true if an element's text content is crawl-analysis meta-text, not a real UI element.\n * These are annotations stored by the accessibility auditor, not actual DOM elements.\n */\nfunction isMetaAnnotation(el: { role?: string | null; meaning?: string | null; text?: string | null }): boolean {\n  const t = (el.meaning ?? el.text ?? '').trim();\n  if (!t) return false;\n  // Filter crawl-analysis annotations\n  if (META_ANNOTATION_RE.test(t)) return true;\n  // Filter very long text strings with no interactive role \u2014 these are paragraph/body copy,\n  // not useful UI locators. Threshold: 120 chars with no testId/label/role\n  const interactiveRoles = new Set(['button', 'link', 'textbox', 'combobox', 'checkbox', 'radio', 'tab', 'option', 'menuitem']);\n  if (t.length > 120 && !interactiveRoles.has((el.role ?? '').toLowerCase())) return true;\n  // Filter CSS variable dumps (contain multiple --var-name patterns)\n  if ((t.match(/--[\\w-]+/g) ?? []).length > 3) return true;\n  return false;\n}\n\n/**\n * Convert an array of DB Element records to deduplicated Playwright locators.\n * Deduplicates by key \u2014 higher confidence wins.\n * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.\n */\nexport function elementsToPlaywrightLocators(elements: Array<{\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n  isInteractive?: boolean | null;\n  ariaState?: Record<string, unknown> | null;\n}>): PlaywrightLocator[] {\n  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };\n  const byKey = new Map<string, PlaywrightLocator>();\n\n  for (const el of elements) {\n    // Skip crawl-analysis meta-annotations \u2014 these are not real page elements\n    if (isMetaAnnotation(el)) continue;\n\n    const loc = elementToPlaywrightLocator(el);\n    if (!loc) continue;\n    const existing = byKey.get(loc.key);\n    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {\n      byKey.set(loc.key, loc);\n    }\n  }\n\n  return Array.from(byKey.values());\n}\n\n/**\n * Detect whether a set of locators was extracted from an auth-wall page.\n * Returns a warning message if auth-wall detected, null otherwise.\n * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.\n */\nexport function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null {\n  if (locators.length === 0) return null;\n  const lowCount = locators.filter(l => l.confidence === 'low').length;\n  const hasAuthText = locators.some(l => AUTH_WALL_TEXT_RE.test(l.description));\n  const highCount = locators.filter(l => l.confidence === 'high').length;\n  if (hasAuthText && highCount === 0 && lowCount / locators.length > 0.5) {\n    return 'These locators appear to be from a login/access wall, not the target page. The page requires authentication. Add credentials in Project Settings and re-crawl to get accurate locators for the protected content.';\n  }\n  return null;\n}\n\n/**\n * Format locators as a block for injection into LLM script generation prompts.\n * Returns empty string if no locators.\n */\nexport function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {\n  if (locators.length === 0) return '';\n  const lines = locators.map(l =>\n    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`\n  );\n  return [\n    'EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):',\n    ...lines,\n  ].join('\\n');\n}\n"],
+  "mappings": "AAeA,SAAS,WAAW,OAAwC;AAC1D,MAAI,CAAC,MAAO,QAAO,CAAC;AACpB,MAAI;AACF,UAAM,SAAS,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;AAC/D,QAAI,UAAU,OAAO,WAAW,SAAU,QAAO;AAAA,EACnD,QAAQ;AAAA,EAAC;AACT,SAAO,CAAC;AACV;AAGA,SAAS,QAAQ,MAAqC;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,IAAI,KAAK,YAAY,EAAE,KAAK;AAClC,MAAI,MAAM,YAAY,MAAM,YAAY,MAAM,QAAS,QAAO;AAC9D,MAAI,MAAM,UAAU,MAAM,OAAO,MAAM,SAAU,QAAO;AACxD,MAAI,MAAM,aAAa,MAAM,WAAW,MAAM,UAAU,MAAM,WAAW,MAAM,cAAc,MAAM,SAAS,MAAM,YAAY,MAAM,SAAU,QAAO;AACrJ,MAAI,MAAM,cAAc,MAAM,YAAY,MAAM,WAAY,QAAO;AACnE,MAAI,MAAM,WAAY,QAAO;AAC7B,MAAI,MAAM,QAAS,QAAO;AAC1B,MAAI,MAAM,aAAa,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,KAAM,QAAO;AAChH,MAAI,MAAM,SAAS,MAAM,QAAS,QAAO;AACzC,MAAI,MAAM,SAAS,MAAM,aAAc,QAAO;AAC9C,SAAO;AACT;AAGA,SAAS,MAAM,MAAsB;AACnC,SAAO,KACJ,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,WAAW,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EAC5C,QAAQ,QAAQ,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EACzC,QAAQ,OAAO,EAAE,EACjB,MAAM,GAAG,EAAE,KACT;AACP;AAGO,SAAS,2BAA2B,SAOd;AAC3B,QAAM,QAAQ,WAAW,QAAQ,KAAK;AACtC,QAAM,SAAS,MAAM,aAAa;AAClC,QAAM,YAAY,MAAM,YAAY,KAAK,QAAQ;AACjD,QAAM,KAAK,MAAM,IAAI;AACrB,QAAM,cAAc,MAAM,aAAa;AACvC,QAAM,OAAO,MAAM,MAAM;AACzB,QAAM,QAAQ,QAAQ,QAAQ,IAAI,KAAK;AACvC,QAAM,WAAW,QAAQ,WAAW,IAAI,KAAK;AAC7C,QAAM,OAAO,QAAQ,QAAQ,IAAI;AACjC,QAAM,YAAoC,CAAC;AAC3C,MAAI,QAAQ,aAAa,OAAO,QAAQ,cAAc,UAAU;AAC9D,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,QAAQ,SAAS,GAAG;AACtD,UAAI,OAAO,MAAM,SAAU,WAAU,CAAC,IAAI;AAAA,IAC5C;AAAA,EACF;AAEA,QAAM,QAAQ,WAAW,QAAQ,UAAU,aAAa,eAAe,MAAM,QAAQ;AACrF,QAAM,MAAM,MAAM,KAAK;AAGvB,MAAI,QAAQ;AACV,WAAO;AAAA,MACL;AAAA,MACA,YAAY,oBAAoB,KAAK,UAAU,MAAM,CAAC;AAAA,MACtD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,SAAS,aAAa,SAAS,YAAY;AAC7C,QAAI,WAAW;AACb,aAAO;AAAA,QACL;AAAA,QACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,QACxD,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AACA,QAAI,aAAa;AACf,aAAO;AAAA,QACL;AAAA,QACA,YAAY,yBAAyB,KAAK,UAAU,WAAW,CAAC;AAAA,QAChE,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAGA,OAAK,SAAS,cAAc,SAAS,YAAY,SAAS;AACxD,UAAM,YAAY,UAAU,SAAS;AACrC,UAAM,WAAW,cAAc,SAAS,aAAa,KAAK,UAAU,OAAO,CAAC,sBAAsB,cAAc,UAAU,aAAa,KAAK,UAAU,OAAO,CAAC,uBAAuB,aAAa,KAAK,UAAU,OAAO,CAAC;AACzN,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,IAAI,CAAC,GAAG,QAAQ;AAAA,MAC7D,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AACA,OAAK,SAAS,cAAc,SAAS,YAAY,WAAW;AAC1D,WAAO;AAAA,MACL;AAAA,MACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,MACxD,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,QAAM,eAAe,CAAC,OAAO,UAAU,YAAY,YAAY,oBAAoB,iBAAiB,UAAU;AAC9G,MAAI,aAAa,SAAS,QAAQ,EAAE,KAAK,SAAS;AAChD,UAAM,aAAa,UAAU,UAAU;AACvC,UAAM,YAAY,UAAU,SAAS;AACrC,UAAM,WAAW,eAAe,SAAS,aAAa,KAAK,UAAU,OAAO,CAAC,uBAAuB,YAAY,aAAa,KAAK,UAAU,OAAO,CAAC,OAAO,aAAa,KAAK,UAAU,OAAO,CAAC;AAC/L,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,IAAI,CAAC,GAAG,QAAQ;AAAA,MAC7D,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,OAAK,SAAS,YAAY,SAAS,WAAW,SAAS;AACrD,UAAM,YAAY,UAAU,SAAS;AACrC,UAAM,WAAW,cAAc,SAAS,aAAa,KAAK,UAAU,OAAO,CAAC,sBAAsB,cAAc,UAAU,aAAa,KAAK,UAAU,OAAO,CAAC,uBAAuB,aAAa,KAAK,UAAU,OAAO,CAAC;AACzN,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,IAAI,CAAC,GAAG,QAAQ;AAAA,MAC7D,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,SAAS,cAAc,WAAW,OAAO;AAC3C,UAAM,QAAQ,WAAW;AACzB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,qCAAqC,KAAK,UAAU,KAAK,CAAC;AAAA,MACtE,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,QAAQ,QAAQ,CAAC;AAAA,MAC5D,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW,QAAQ,QAAQ;AAAA,IAC1C;AAAA,EACF;AAGA,MAAI,IAAI;AACN,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,MAAM,EAAE,CAAC;AAAA,MACpD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,CAAC,CAAC;AAAA,MAC/C,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAEA,SAAO;AACT;AAIA,MAAM,qBAAqB;AAG3B,MAAM,oBAAoB;AAM1B,SAAS,iBAAiB,IAAsF;AAC9G,QAAM,KAAK,GAAG,WAAW,GAAG,QAAQ,IAAI,KAAK;AAC7C,MAAI,CAAC,EAAG,QAAO;AAEf,MAAI,mBAAmB,KAAK,CAAC,EAAG,QAAO;AAGvC,QAAM,mBAAmB,oBAAI,IAAI,CAAC,UAAU,QAAQ,WAAW,YAAY,YAAY,SAAS,OAAO,UAAU,UAAU,CAAC;AAC5H,MAAI,EAAE,SAAS,OAAO,CAAC,iBAAiB,KAAK,GAAG,QAAQ,IAAI,YAAY,CAAC,EAAG,QAAO;AAEnF,OAAK,EAAE,MAAM,WAAW,KAAK,CAAC,GAAG,SAAS,EAAG,QAAO;AACpD,SAAO;AACT;AAOO,SAAS,6BAA6B,UAQpB;AACvB,QAAM,kBAAkB,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,EAAE;AACrD,QAAM,QAAQ,oBAAI,IAA+B;AAEjD,aAAW,MAAM,UAAU;AAEzB,QAAI,iBAAiB,EAAE,EAAG;AAE1B,UAAM,MAAM,2BAA2B,EAAE;AACzC,QAAI,CAAC,IAAK;AACV,UAAM,WAAW,MAAM,IAAI,IAAI,GAAG;AAClC,QAAI,CAAC,YAAY,gBAAgB,IAAI,UAAU,IAAI,gBAAgB,SAAS,UAAU,GAAG;AACvF,YAAM,IAAI,IAAI,KAAK,GAAG;AAAA,IACxB;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,MAAM,OAAO,CAAC;AAClC;AAOO,SAAS,uBAAuB,UAA8C;AACnF,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,WAAW,SAAS,OAAO,OAAK,EAAE,eAAe,KAAK,EAAE;AAC9D,QAAM,cAAc,SAAS,KAAK,OAAK,kBAAkB,KAAK,EAAE,WAAW,CAAC;AAC5E,QAAM,YAAY,SAAS,OAAO,OAAK,EAAE,eAAe,MAAM,EAAE;AAChE,MAAI,eAAe,cAAc,KAAK,WAAW,SAAS,SAAS,KAAK;AACtE,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAMO,SAAS,wBAAwB,UAAuC;AAC7E,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,QAAQ,SAAS;AAAA,IAAI,OACzB,KAAK,EAAE,GAAG,KAAK,EAAE,UAAU,QAAQ,EAAE,WAAW,KAAK,EAAE,UAAU;AAAA,EACnE;AACA,SAAO;AAAA,IACL;AAAA,IACA,GAAG;AAAA,EACL,EAAE,KAAK,IAAI;AACb;",
+  "names": []
+}

package/dist/logger.d.ts

@@ -0,0 +1,2 @@
+import pino from 'pino';
+export declare const logger: pino.Logger<never, boolean>;

package/dist/logger.js

@@ -0,0 +1,10 @@
+import pino from "pino";
+const logger = pino({
+  level: process.env.LOG_LEVEL ?? "info",
+  ...process.env.NODE_ENV === "development" ? { transport: { target: "pino-pretty", options: { colorize: true } } } : {},
+  base: { service: "crawler" }
+});
+export {
+  logger
+};
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/logger.ts"],
+  "sourcesContent": ["import pino from 'pino';\n\nexport const logger = pino({\n  level: process.env.LOG_LEVEL ?? 'info',\n  ...(process.env.NODE_ENV === 'development'\n    ? { transport: { target: 'pino-pretty', options: { colorize: true } } }\n    : {}),\n  base: { service: 'crawler' },\n});\n"],
+  "mappings": "AAAA,OAAO,UAAU;AAEV,MAAM,SAAS,KAAK;AAAA,EACzB,OAAO,QAAQ,IAAI,aAAa;AAAA,EAChC,GAAI,QAAQ,IAAI,aAAa,gBACzB,EAAE,WAAW,EAAE,QAAQ,eAAe,SAAS,EAAE,UAAU,KAAK,EAAE,EAAE,IACpE,CAAC;AAAA,EACL,MAAM,EAAE,SAAS,UAAU;AAC7B,CAAC;",
+  "names": []
+}

package/dist/markdown-extract.d.ts

@@ -0,0 +1,8 @@
+export interface ExtractedMarkdown {
+    markdown: string;
+    method: 'readability' | 'full-page';
+    title: string | null;
+    wordCount: number;
+}
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export declare function extractMarkdown(html: string, url: string): ExtractedMarkdown | null;

package/dist/markdown-extract.js

@@ -0,0 +1,54 @@
+import { parseHTML } from "linkedom";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+turndown.remove(["script", "style", "noscript", "svg", "iframe"]);
+function stripNoiseElements(document) {
+  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+  noisy.forEach((el) => el.remove());
+}
+function extractMarkdown(html, url) {
+  try {
+    const { document } = parseHTML(html);
+    stripNoiseElements(document);
+    let readabilityResult = null;
+    try {
+      const { document: readerDoc } = parseHTML(html);
+      stripNoiseElements(readerDoc);
+      readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
+    } catch {
+      readabilityResult = null;
+    }
+    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+      const markdown2 = turndown.turndown(readabilityResult.content);
+      return {
+        markdown: `# ${readabilityResult.title || document.title || url}
+
+${markdown2}`,
+        method: "readability",
+        title: readabilityResult.title || document.title || null,
+        wordCount: readabilityResult.textContent.trim().split(/\s+/).length
+      };
+    }
+    const body = document.body;
+    if (!body) return null;
+    const markdown = turndown.turndown(body.innerHTML || "");
+    const text = markdown.trim();
+    if (!text) return null;
+    const title = document.title || null;
+    return {
+      markdown: title ? `# ${title}
+
+${markdown}` : markdown,
+      method: "full-page",
+      title,
+      wordCount: text.split(/\s+/).length
+    };
+  } catch {
+    return null;
+  }
+}
+export {
+  extractMarkdown
+};
+//# sourceMappingURL=markdown-extract.js.map

package/dist/markdown-extract.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/markdown-extract.ts"],
+  "sourcesContent": ["/**\n * HTML \u2192 Markdown extraction for crawled pages \u2014 matches crawl4AI's markdown\n * output capability. Two-tier, mirroring crawl4AI's \"markdown\" vs \"fit markdown\":\n *\n *  - \"fit\" markdown: Readability's best guess at the primary content (title +\n *    article body), stripped of nav/ads/chrome. Ideal for content/marketing\n *    pages and RAG chunking. null when Readability finds no article (most\n *    app UI screens \u2014 dashboards, forms \u2014 legitimately have none).\n *  - \"full\" markdown: the whole visible page converted straight to Markdown,\n *    always present. The fallback for app screens where there's no \"article\".\n *\n * Runs entirely on an already-captured `page.content()` string \u2014 no extra\n * network fetch, no extra page navigation.\n */\nimport { parseHTML } from 'linkedom';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\n\nconst turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });\nturndown.remove(['script', 'style', 'noscript', 'svg', 'iframe'] as unknown as (keyof HTMLElementTagNameMap)[]);\n\nexport interface ExtractedMarkdown {\n  markdown: string;\n  method: 'readability' | 'full-page';\n  title: string | null;\n  wordCount: number;\n}\n\nfunction stripNoiseElements(document: Document): void {\n  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden=\"true\"]');\n  noisy.forEach((el: any) => el.remove());\n}\n\n/** Best-effort HTML\u2192Markdown for one crawled page. Never throws. */\nexport function extractMarkdown(html: string, url: string): ExtractedMarkdown | null {\n  try {\n    const { document } = parseHTML(html);\n    stripNoiseElements(document);\n\n    let readabilityResult: { title: string; content: string; textContent: string } | null = null;\n    try {\n      // Readability mutates the DOM it's given \u2014 clone via re-parsing so the\n      // full-page fallback below still has an intact document.\n      const { document: readerDoc } = parseHTML(html);\n      stripNoiseElements(readerDoc);\n      readabilityResult = new Readability(readerDoc as any, { charThreshold: 50 }).parse() as any;\n    } catch {\n      readabilityResult = null;\n    }\n\n    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {\n      const markdown = turndown.turndown(readabilityResult.content);\n      return {\n        markdown: `# ${readabilityResult.title || document.title || url}\\n\\n${markdown}`,\n        method: 'readability',\n        title: readabilityResult.title || document.title || null,\n        wordCount: readabilityResult.textContent.trim().split(/\\s+/).length,\n      };\n    }\n\n    const body = document.body;\n    if (!body) return null;\n    const markdown = turndown.turndown(body.innerHTML || '');\n    const text = markdown.trim();\n    if (!text) return null;\n    const title = document.title || null;\n    return {\n      markdown: title ? `# ${title}\\n\\n${markdown}` : markdown,\n      method: 'full-page',\n      title,\n      wordCount: text.split(/\\s+/).length,\n    };\n  } catch {\n    return null;\n  }\n}\n"],
+  "mappings": "AAcA,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB;AAC5B,OAAO,qBAAqB;AAE5B,MAAM,WAAW,IAAI,gBAAgB,EAAE,cAAc,OAAO,gBAAgB,SAAS,CAAC;AACtF,SAAS,OAAO,CAAC,UAAU,SAAS,YAAY,OAAO,QAAQ,CAA+C;AAS9G,SAAS,mBAAmB,UAA0B;AACpD,QAAM,QAAQ,SAAS,iBAAiB,4DAA4D;AACpG,QAAM,QAAQ,CAAC,OAAY,GAAG,OAAO,CAAC;AACxC;AAGO,SAAS,gBAAgB,MAAc,KAAuC;AACnF,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,uBAAmB,QAAQ;AAE3B,QAAI,oBAAoF;AACxF,QAAI;AAGF,YAAM,EAAE,UAAU,UAAU,IAAI,UAAU,IAAI;AAC9C,yBAAmB,SAAS;AAC5B,0BAAoB,IAAI,YAAY,WAAkB,EAAE,eAAe,GAAG,CAAC,EAAE,MAAM;AAAA,IACrF,QAAQ;AACN,0BAAoB;AAAA,IACtB;AAEA,QAAI,mBAAmB,WAAW,kBAAkB,YAAY,KAAK,EAAE,UAAU,IAAI;AACnF,YAAMA,YAAW,SAAS,SAAS,kBAAkB,OAAO;AAC5D,aAAO;AAAA,QACL,UAAU,KAAK,kBAAkB,SAAS,SAAS,SAAS,GAAG;AAAA;AAAA,EAAOA,SAAQ;AAAA,QAC9E,QAAQ;AAAA,QACR,OAAO,kBAAkB,SAAS,SAAS,SAAS;AAAA,QACpD,WAAW,kBAAkB,YAAY,KAAK,EAAE,MAAM,KAAK,EAAE;AAAA,MAC/D;AAAA,IACF;AAEA,UAAM,OAAO,SAAS;AACtB,QAAI,CAAC,KAAM,QAAO;AAClB,UAAM,WAAW,SAAS,SAAS,KAAK,aAAa,EAAE;AACvD,UAAM,OAAO,SAAS,KAAK;AAC3B,QAAI,CAAC,KAAM,QAAO;AAClB,UAAM,QAAQ,SAAS,SAAS;AAChC,WAAO;AAAA,MACL,UAAU,QAAQ,KAAK,KAAK;AAAA;AAAA,EAAO,QAAQ,KAAK;AAAA,MAChD,QAAQ;AAAA,MACR;AAAA,MACA,WAAW,KAAK,MAAM,KAAK,EAAE;AAAA,IAC/B;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": ["markdown"]
+}

package/dist/multi-context-handler.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Multi-context browser handling — detects and records new tabs, iframes,
+ * popups, and file downloads triggered during a page crawl.
+ * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
+ * third-party content, and downloadable resources that single-page HTTP
+ * crawlers would miss entirely.
+ */
+import type { Page, BrowserContext } from 'playwright';
+export interface NewTabRecord {
+    url: string;
+    linkText: string;
+    isInternal: boolean;
+    httpStatus?: number;
+}
+export interface IframeRecord {
+    src: string;
+    crossOrigin: boolean;
+    sandboxed: boolean;
+    sandboxValue: string | null;
+    accessible: boolean;
+    contentSample: string;
+    hasForm: boolean;
+}
+export interface PopupRecord {
+    url: string;
+    triggerLabel: string;
+    isOAuthLike: boolean;
+}
+export interface DownloadRecord {
+    filename: string;
+    downloadUrl: string;
+    triggerLabel: string;
+}
+export interface MultiContextResult {
+    newTabs: NewTabRecord[];
+    iframes: IframeRecord[];
+    popups: PopupRecord[];
+    downloads: DownloadRecord[];
+}
+/**
+ * Attaches a listener to the BrowserContext that records every new tab
+ * opened while the listener is active. Returns an unsubscribe function.
+ *
+ * Call this before navigating / clicking so that tabs opened during the
+ * crawl are captured, then call the returned function to stop listening.
+ */
+export declare function attachNewTabListener(context: BrowserContext, seedUrl: string, result: MultiContextResult): () => void;
+/**
+ * Probes the current page for multi-context browser events: iframes,
+ * popup windows, and file downloads.
+ *
+ * New-tab tracking is intentionally excluded here — it must be set up at
+ * the crawler level (before navigation) using `attachNewTabListener`.
+ */
+export declare function probeMultiContext(page: Page, context: BrowserContext, seedUrl: string): Promise<MultiContextResult>;

package/dist/multi-context-handler.js

@@ -0,0 +1,182 @@
+async function enumerateIframes(page, seedUrl) {
+  const records = [];
+  try {
+    const seedHost = new URL(seedUrl).hostname;
+    const iframeLocators = await page.locator("iframe").all();
+    const limited = iframeLocators.slice(0, 20);
+    for (let idx = 0; idx < limited.length; idx++) {
+      try {
+        const loc = limited[idx];
+        const src = await loc.getAttribute("src") ?? "";
+        const sandboxValue = await loc.getAttribute("sandbox");
+        const sandboxed = sandboxValue !== null;
+        let crossOrigin = false;
+        if (src) {
+          try {
+            const iframeHost = new URL(src, seedUrl).hostname;
+            crossOrigin = iframeHost !== seedHost;
+          } catch {
+            crossOrigin = false;
+          }
+        }
+        let accessible = false;
+        let contentSample = "";
+        let hasForm = false;
+        try {
+          const fl = page.frameLocator("iframe").nth(idx);
+          const body = await fl.locator("body").textContent({ timeout: 2e3 });
+          if (body !== null) {
+            accessible = true;
+            contentSample = body.trim().slice(0, 300);
+          }
+          const formCount = await fl.locator("form, input, textarea, select").count();
+          hasForm = formCount > 0;
+        } catch {
+        }
+        records.push({
+          src,
+          crossOrigin,
+          sandboxed,
+          sandboxValue,
+          accessible,
+          contentSample,
+          hasForm
+        });
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return records;
+}
+async function detectPopups(page, result) {
+  try {
+    const candidates = [
+      ...await page.locator('[onclick*="window.open"]').all(),
+      ...await page.locator('button:has-text("Open in")').all()
+    ];
+    const limited = candidates.slice(0, 3);
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? "";
+        const outcome = await Promise.all([
+          page.waitForEvent("popup", { timeout: 2e3 }),
+          trigger.click()
+        ]).catch(() => null);
+        if (!outcome) continue;
+        const popup = outcome[0];
+        if (!popup) continue;
+        const url = popup.url();
+        let isOAuthLike = false;
+        try {
+          const host = new URL(url).hostname;
+          isOAuthLike = /google|github|microsoft|okta|auth0/.test(host);
+        } catch {
+        }
+        result.popups.push({ url, triggerLabel, isOAuthLike });
+        await popup.close().catch(() => {
+        });
+      } catch {
+      }
+    }
+  } catch {
+  }
+}
+async function detectDownloads(page, result) {
+  try {
+    const candidates = [
+      ...await page.locator("a[download]").all(),
+      ...await page.locator('button:has-text("Export")').all(),
+      ...await page.locator('button:has-text("Download")').all()
+    ];
+    const limited = candidates.slice(0, 3);
+    for (const trigger of limited) {
+      try {
+        const triggerLabel = (await trigger.textContent())?.trim() ?? "";
+        const outcome = await Promise.all([
+          page.waitForEvent("download", { timeout: 2e3 }),
+          trigger.click()
+        ]).catch(() => null);
+        if (!outcome) continue;
+        const download = outcome[0];
+        if (!download) continue;
+        const filename = download.suggestedFilename();
+        const downloadUrl = download.url();
+        await download.cancel().catch(() => {
+        });
+        result.downloads.push({ filename, downloadUrl, triggerLabel });
+      } catch {
+      }
+    }
+  } catch {
+  }
+}
+function attachNewTabListener(context, seedUrl, result) {
+  let seedHost = "";
+  try {
+    seedHost = new URL(seedUrl).hostname;
+  } catch {
+  }
+  const handler = async (newPage) => {
+    try {
+      await newPage.waitForLoadState("networkidle", { timeout: 1e4 }).catch(() => {
+      });
+      const url = newPage.url();
+      if (!url || url === "about:blank") {
+        await newPage.close().catch(() => {
+        });
+        return;
+      }
+      let isInternal = false;
+      let httpStatus;
+      try {
+        const tabHost = new URL(url).hostname;
+        isInternal = tabHost === seedHost;
+      } catch {
+      }
+      if (!isInternal) {
+        try {
+          const controller = new AbortController();
+          const timer = setTimeout(() => controller.abort(), 3e3);
+          const resp = await fetch(url, { method: "HEAD", signal: controller.signal });
+          clearTimeout(timer);
+          httpStatus = resp.status;
+        } catch {
+        }
+      }
+      result.newTabs.push({ url, linkText: "", isInternal, httpStatus });
+      await newPage.close().catch(() => {
+      });
+    } catch {
+    }
+  };
+  context.on("page", handler);
+  return () => context.off("page", handler);
+}
+async function probeMultiContext(page, context, seedUrl) {
+  void context;
+  const result = {
+    newTabs: [],
+    iframes: [],
+    popups: [],
+    downloads: []
+  };
+  try {
+    result.iframes = await enumerateIframes(page, seedUrl);
+  } catch {
+  }
+  try {
+    await detectPopups(page, result);
+  } catch {
+  }
+  try {
+    await detectDownloads(page, result);
+  } catch {
+  }
+  return result;
+}
+export {
+  attachNewTabListener,
+  probeMultiContext
+};
+//# sourceMappingURL=multi-context-handler.js.map

package/dist/multi-context-handler.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/multi-context-handler.ts"],
+  "sourcesContent": ["/**\n * Multi-context browser handling \u2014 detects and records new tabs, iframes,\n * popups, and file downloads triggered during a page crawl.\n * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded\n * third-party content, and downloadable resources that single-page HTTP\n * crawlers would miss entirely.\n */\n\nimport type { Page, BrowserContext } from 'playwright';\n\nexport interface NewTabRecord {\n  url: string;\n  linkText: string;\n  isInternal: boolean;\n  httpStatus?: number;\n}\n\nexport interface IframeRecord {\n  src: string;\n  crossOrigin: boolean;\n  sandboxed: boolean;\n  sandboxValue: string | null;\n  accessible: boolean;\n  contentSample: string;\n  hasForm: boolean;\n}\n\nexport interface PopupRecord {\n  url: string;\n  triggerLabel: string;\n  isOAuthLike: boolean;\n}\n\nexport interface DownloadRecord {\n  filename: string;\n  downloadUrl: string;\n  triggerLabel: string;\n}\n\nexport interface MultiContextResult {\n  newTabs: NewTabRecord[];\n  iframes: IframeRecord[];\n  popups: PopupRecord[];\n  downloads: DownloadRecord[];\n}\n\n// ---------------------------------------------------------------------------\n// Internal helpers\n// ---------------------------------------------------------------------------\n\nasync function enumerateIframes(\n  page: Page,\n  seedUrl: string,\n): Promise<IframeRecord[]> {\n  const records: IframeRecord[] = [];\n  try {\n    const seedHost = new URL(seedUrl).hostname;\n    const iframeLocators = await page.locator('iframe').all();\n    const limited = iframeLocators.slice(0, 20);\n\n    for (let idx = 0; idx < limited.length; idx++) {\n      try {\n        const loc = limited[idx];\n        const src = (await loc.getAttribute('src')) ?? '';\n        const sandboxValue = await loc.getAttribute('sandbox');\n        const sandboxed = sandboxValue !== null;\n\n        let crossOrigin = false;\n        if (src) {\n          try {\n            const iframeHost = new URL(src, seedUrl).hostname;\n            crossOrigin = iframeHost !== seedHost;\n          } catch {\n            crossOrigin = false;\n          }\n        }\n\n        let accessible = false;\n        let contentSample = '';\n        let hasForm = false;\n\n        try {\n          const fl = page.frameLocator('iframe').nth(idx);\n          const body = await fl.locator('body').textContent({ timeout: 2000 });\n          if (body !== null) {\n            accessible = true;\n            contentSample = body.trim().slice(0, 300);\n          }\n          const formCount = await fl.locator('form, input, textarea, select').count();\n          hasForm = formCount > 0;\n        } catch {\n          // cross-origin or sandboxed \u2014 not accessible from parent frame\n        }\n\n        records.push({\n          src,\n          crossOrigin,\n          sandboxed,\n          sandboxValue,\n          accessible,\n          contentSample,\n          hasForm,\n        });\n      } catch {\n        // skip individual iframe errors\n      }\n    }\n  } catch {\n    // skip entirely if locator fails\n  }\n  return records;\n}\n\nasync function detectPopups(page: Page, result: MultiContextResult): Promise<void> {\n  try {\n    const candidates = [\n      ...await page.locator('[onclick*=\"window.open\"]').all(),\n      ...await page.locator('button:has-text(\"Open in\")').all(),\n    ];\n    const limited = candidates.slice(0, 3);\n\n    for (const trigger of limited) {\n      try {\n        const triggerLabel = (await trigger.textContent())?.trim() ?? '';\n        const outcome = await Promise.all([\n          page.waitForEvent('popup', { timeout: 2000 }),\n          trigger.click(),\n        ]).catch(() => null);\n\n        if (!outcome) continue;\n        const popup = outcome[0] as Page;\n        if (!popup) continue;\n\n        const url = popup.url();\n        let isOAuthLike = false;\n        try {\n          const host = new URL(url).hostname;\n          isOAuthLike = /google|github|microsoft|okta|auth0/.test(host);\n        } catch {\n          // not a valid URL\n        }\n\n        result.popups.push({ url, triggerLabel, isOAuthLike });\n        await popup.close().catch(() => {});\n      } catch {\n        // skip this trigger\n      }\n    }\n  } catch {\n    // skip popup detection entirely\n  }\n}\n\nasync function detectDownloads(page: Page, result: MultiContextResult): Promise<void> {\n  try {\n    const candidates = [\n      ...await page.locator('a[download]').all(),\n      ...await page.locator('button:has-text(\"Export\")').all(),\n      ...await page.locator('button:has-text(\"Download\")').all(),\n    ];\n    const limited = candidates.slice(0, 3);\n\n    for (const trigger of limited) {\n      try {\n        const triggerLabel = (await trigger.textContent())?.trim() ?? '';\n        const outcome = await Promise.all([\n          page.waitForEvent('download', { timeout: 2000 }),\n          trigger.click(),\n        ]).catch(() => null);\n\n        if (!outcome) continue;\n        const download = outcome[0] as unknown as import('playwright').Download;\n        if (!download) continue;\n\n        const filename = download.suggestedFilename();\n        const downloadUrl = download.url();\n        await download.cancel().catch(() => {});\n\n        result.downloads.push({ filename, downloadUrl, triggerLabel });\n      } catch {\n        // skip this trigger\n      }\n    }\n  } catch {\n    // skip download detection entirely\n  }\n}\n\n// ---------------------------------------------------------------------------\n// Exported functions\n// ---------------------------------------------------------------------------\n\n/**\n * Attaches a listener to the BrowserContext that records every new tab\n * opened while the listener is active. Returns an unsubscribe function.\n *\n * Call this before navigating / clicking so that tabs opened during the\n * crawl are captured, then call the returned function to stop listening.\n */\nexport function attachNewTabListener(\n  context: BrowserContext,\n  seedUrl: string,\n  result: MultiContextResult,\n): () => void {\n  let seedHost = '';\n  try {\n    seedHost = new URL(seedUrl).hostname;\n  } catch {\n    // leave empty \u2014 all tabs will be treated as external\n  }\n\n  const handler = async (newPage: Page) => {\n    try {\n      await newPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});\n      const url = newPage.url();\n\n      if (!url || url === 'about:blank') {\n        await newPage.close().catch(() => {});\n        return;\n      }\n\n      let isInternal = false;\n      let httpStatus: number | undefined;\n\n      try {\n        const tabHost = new URL(url).hostname;\n        isInternal = tabHost === seedHost;\n      } catch {\n        // non-parseable URL \u2014 treat as external\n      }\n\n      if (!isInternal) {\n        try {\n          const controller = new AbortController();\n          const timer = setTimeout(() => controller.abort(), 3000);\n          const resp = await fetch(url, { method: 'HEAD', signal: controller.signal });\n          clearTimeout(timer);\n          httpStatus = resp.status;\n        } catch {\n          // network unreachable or CORS \u2014 leave httpStatus undefined\n        }\n      }\n\n      // linkText is not available at context level; caller fills it if needed\n      result.newTabs.push({ url, linkText: '', isInternal, httpStatus });\n      await newPage.close().catch(() => {});\n    } catch {\n      // swallow handler errors so the listener stays alive\n    }\n  };\n\n  context.on('page', handler);\n  return () => context.off('page', handler);\n}\n\n/**\n * Probes the current page for multi-context browser events: iframes,\n * popup windows, and file downloads.\n *\n * New-tab tracking is intentionally excluded here \u2014 it must be set up at\n * the crawler level (before navigation) using `attachNewTabListener`.\n */\nexport async function probeMultiContext(\n  page: Page,\n  context: BrowserContext,\n  seedUrl: string,\n): Promise<MultiContextResult> {\n  void context; // reserved for future use (e.g. routing overrides)\n\n  const result: MultiContextResult = {\n    newTabs: [],\n    iframes: [],\n    popups: [],\n    downloads: [],\n  };\n\n  try {\n    result.iframes = await enumerateIframes(page, seedUrl);\n  } catch {\n    // keep empty\n  }\n\n  try {\n    await detectPopups(page, result);\n  } catch {\n    // keep empty\n  }\n\n  try {\n    await detectDownloads(page, result);\n  } catch {\n    // keep empty\n  }\n\n  return result;\n}\n"],
+  "mappings": "AAkDA,eAAe,iBACb,MACA,SACyB;AACzB,QAAM,UAA0B,CAAC;AACjC,MAAI;AACF,UAAM,WAAW,IAAI,IAAI,OAAO,EAAE;AAClC,UAAM,iBAAiB,MAAM,KAAK,QAAQ,QAAQ,EAAE,IAAI;AACxD,UAAM,UAAU,eAAe,MAAM,GAAG,EAAE;AAE1C,aAAS,MAAM,GAAG,MAAM,QAAQ,QAAQ,OAAO;AAC7C,UAAI;AACF,cAAM,MAAM,QAAQ,GAAG;AACvB,cAAM,MAAO,MAAM,IAAI,aAAa,KAAK,KAAM;AAC/C,cAAM,eAAe,MAAM,IAAI,aAAa,SAAS;AACrD,cAAM,YAAY,iBAAiB;AAEnC,YAAI,cAAc;AAClB,YAAI,KAAK;AACP,cAAI;AACF,kBAAM,aAAa,IAAI,IAAI,KAAK,OAAO,EAAE;AACzC,0BAAc,eAAe;AAAA,UAC/B,QAAQ;AACN,0BAAc;AAAA,UAChB;AAAA,QACF;AAEA,YAAI,aAAa;AACjB,YAAI,gBAAgB;AACpB,YAAI,UAAU;AAEd,YAAI;AACF,gBAAM,KAAK,KAAK,aAAa,QAAQ,EAAE,IAAI,GAAG;AAC9C,gBAAM,OAAO,MAAM,GAAG,QAAQ,MAAM,EAAE,YAAY,EAAE,SAAS,IAAK,CAAC;AACnE,cAAI,SAAS,MAAM;AACjB,yBAAa;AACb,4BAAgB,KAAK,KAAK,EAAE,MAAM,GAAG,GAAG;AAAA,UAC1C;AACA,gBAAM,YAAY,MAAM,GAAG,QAAQ,+BAA+B,EAAE,MAAM;AAC1E,oBAAU,YAAY;AAAA,QACxB,QAAQ;AAAA,QAER;AAEA,gBAAQ,KAAK;AAAA,UACX;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF,CAAC;AAAA,MACH,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAY,QAA2C;AACjF,MAAI;AACF,UAAM,aAAa;AAAA,MACjB,GAAG,MAAM,KAAK,QAAQ,0BAA0B,EAAE,IAAI;AAAA,MACtD,GAAG,MAAM,KAAK,QAAQ,4BAA4B,EAAE,IAAI;AAAA,IAC1D;AACA,UAAM,UAAU,WAAW,MAAM,GAAG,CAAC;AAErC,eAAW,WAAW,SAAS;AAC7B,UAAI;AACF,cAAM,gBAAgB,MAAM,QAAQ,YAAY,IAAI,KAAK,KAAK;AAC9D,cAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,UAChC,KAAK,aAAa,SAAS,EAAE,SAAS,IAAK,CAAC;AAAA,UAC5C,QAAQ,MAAM;AAAA,QAChB,CAAC,EAAE,MAAM,MAAM,IAAI;AAEnB,YAAI,CAAC,QAAS;AACd,cAAM,QAAQ,QAAQ,CAAC;AACvB,YAAI,CAAC,MAAO;AAEZ,cAAM,MAAM,MAAM,IAAI;AACtB,YAAI,cAAc;AAClB,YAAI;AACF,gBAAM,OAAO,IAAI,IAAI,GAAG,EAAE;AAC1B,wBAAc,qCAAqC,KAAK,IAAI;AAAA,QAC9D,QAAQ;AAAA,QAER;AAEA,eAAO,OAAO,KAAK,EAAE,KAAK,cAAc,YAAY,CAAC;AACrD,cAAM,MAAM,MAAM,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACpC,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AACF;AAEA,eAAe,gBAAgB,MAAY,QAA2C;AACpF,MAAI;AACF,UAAM,aAAa;AAAA,MACjB,GAAG,MAAM,KAAK,QAAQ,aAAa,EAAE,IAAI;AAAA,MACzC,GAAG,MAAM,KAAK,QAAQ,2BAA2B,EAAE,IAAI;AAAA,MACvD,GAAG,MAAM,KAAK,QAAQ,6BAA6B,EAAE,IAAI;AAAA,IAC3D;AACA,UAAM,UAAU,WAAW,MAAM,GAAG,CAAC;AAErC,eAAW,WAAW,SAAS;AAC7B,UAAI;AACF,cAAM,gBAAgB,MAAM,QAAQ,YAAY,IAAI,KAAK,KAAK;AAC9D,cAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,UAChC,KAAK,aAAa,YAAY,EAAE,SAAS,IAAK,CAAC;AAAA,UAC/C,QAAQ,MAAM;AAAA,QAChB,CAAC,EAAE,MAAM,MAAM,IAAI;AAEnB,YAAI,CAAC,QAAS;AACd,cAAM,WAAW,QAAQ,CAAC;AAC1B,YAAI,CAAC,SAAU;AAEf,cAAM,WAAW,SAAS,kBAAkB;AAC5C,cAAM,cAAc,SAAS,IAAI;AACjC,cAAM,SAAS,OAAO,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAEtC,eAAO,UAAU,KAAK,EAAE,UAAU,aAAa,aAAa,CAAC;AAAA,MAC/D,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AACF;AAaO,SAAS,qBACd,SACA,SACA,QACY;AACZ,MAAI,WAAW;AACf,MAAI;AACF,eAAW,IAAI,IAAI,OAAO,EAAE;AAAA,EAC9B,QAAQ;AAAA,EAER;AAEA,QAAM,UAAU,OAAO,YAAkB;AACvC,QAAI;AACF,YAAM,QAAQ,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAChF,YAAM,MAAM,QAAQ,IAAI;AAExB,UAAI,CAAC,OAAO,QAAQ,eAAe;AACjC,cAAM,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACpC;AAAA,MACF;AAEA,UAAI,aAAa;AACjB,UAAI;AAEJ,UAAI;AACF,cAAM,UAAU,IAAI,IAAI,GAAG,EAAE;AAC7B,qBAAa,YAAY;AAAA,MAC3B,QAAQ;AAAA,MAER;AAEA,UAAI,CAAC,YAAY;AACf,YAAI;AACF,gBAAM,aAAa,IAAI,gBAAgB;AACvC,gBAAM,QAAQ,WAAW,MAAM,WAAW,MAAM,GAAG,GAAI;AACvD,gBAAM,OAAO,MAAM,MAAM,KAAK,EAAE,QAAQ,QAAQ,QAAQ,WAAW,OAAO,CAAC;AAC3E,uBAAa,KAAK;AAClB,uBAAa,KAAK;AAAA,QACpB,QAAQ;AAAA,QAER;AAAA,MACF;AAGA,aAAO,QAAQ,KAAK,EAAE,KAAK,UAAU,IAAI,YAAY,WAAW,CAAC;AACjE,YAAM,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACtC,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,UAAQ,GAAG,QAAQ,OAAO;AAC1B,SAAO,MAAM,QAAQ,IAAI,QAAQ,OAAO;AAC1C;AASA,eAAsB,kBACpB,MACA,SACA,SAC6B;AAC7B,OAAK;AAEL,QAAM,SAA6B;AAAA,IACjC,SAAS,CAAC;AAAA,IACV,SAAS,CAAC;AAAA,IACV,QAAQ,CAAC;AAAA,IACT,WAAW,CAAC;AAAA,EACd;AAEA,MAAI;AACF,WAAO,UAAU,MAAM,iBAAiB,MAAM,OAAO;AAAA,EACvD,QAAQ;AAAA,EAER;AAEA,MAAI;AACF,UAAM,aAAa,MAAM,MAAM;AAAA,EACjC,QAAQ;AAAA,EAER;AAEA,MAAI;AACF,UAAM,gBAAgB,MAAM,MAAM;AAAA,EACpC,QAAQ;AAAA,EAER;AAEA,SAAO;AACT;",
+  "names": []
+}