@detiq/crawler 1.1.6 → 1.1.8

package/dist/locator-extractor.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+}>): PlaywrightLocator[];
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;

package/dist/locator-extractor.js

@@ -0,0 +1,163 @@
+function parseNotes(notes) {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === "string" ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === "object") return parsed;
+  } catch {
+  }
+  return {};
+}
+function mapRole(role) {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === "button" || r === "submit" || r === "reset") return "button";
+  if (r === "link" || r === "a" || r === "anchor") return "link";
+  if (r === "textbox" || r === "input" || r === "text" || r === "email" || r === "password" || r === "tel" || r === "number" || r === "search") return "textbox";
+  if (r === "combobox" || r === "select" || r === "dropdown") return "combobox";
+  if (r === "checkbox") return "checkbox";
+  if (r === "radio") return "radio";
+  if (r === "heading" || r === "h1" || r === "h2" || r === "h3" || r === "h4" || r === "h5" || r === "h6") return "heading";
+  if (r === "img" || r === "image") return "img";
+  if (r === "nav" || r === "navigation") return "navigation";
+  return r;
+}
+function toKey(text) {
+  return text.replace(/[^a-zA-Z0-9\s]/g, " ").replace(/\s+(.)/g, (_, c) => c.toUpperCase()).replace(/^(.)/, (_, c) => c.toLowerCase()).replace(/\s/g, "").slice(0, 40) || "element";
+}
+function elementToPlaywrightLocator(element) {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs["data-testid"];
+  const ariaLabel = attrs["aria-label"] || element.meaning;
+  const id = attrs["id"];
+  const placeholder = attrs["placeholder"];
+  const name = attrs["name"];
+  const text = (element.text ?? "").trim();
+  const meaning = (element.meaning ?? "").trim();
+  const role = mapRole(element.role);
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || "element";
+  const key = toKey(label);
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? "element",
+      confidence: "high",
+      source: "testid",
+      description: meaning || testId
+    };
+  }
+  if (role === "textbox" || role === "combobox") {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "label",
+        description: ariaLabel
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "placeholder",
+        description: placeholder
+      };
+    }
+  }
+  if ((role === "checkbox" || role === "radio") && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: "high",
+      source: "label",
+      description: ariaLabel
+    };
+  }
+  if ((role === "button" || role === "link") && meaning) {
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      elementType: role,
+      confidence: "high",
+      source: "role-name",
+      description: meaning
+    };
+  }
+  if (role === "heading" && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: "heading",
+      confidence: "medium",
+      source: "heading",
+      description: name2
+    };
+  }
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || text || element.selector
+    };
+  }
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify("#" + id)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || id
+    };
+  }
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? "text",
+      confidence: "low",
+      source: "text",
+      description: t
+    };
+  }
+  return null;
+}
+function elementsToPlaywrightLocators(elements) {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = /* @__PURE__ */ new Map();
+  for (const el of elements) {
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+  return Array.from(byKey.values());
+}
+function formatLocatorsForPrompt(locators) {
+  if (locators.length === 0) return "";
+  const lines = locators.map(
+    (l) => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    "EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):",
+    ...lines
+  ].join("\n");
+}
+export {
+  elementToPlaywrightLocator,
+  elementsToPlaywrightLocators,
+  formatLocatorsForPrompt
+};
+//# sourceMappingURL=locator-extractor.js.map

package/dist/locator-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/locator-extractor.ts"],
+  "sourcesContent": ["/**\n * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.\n * Priority: data-testid \u2192 aria-label/placeholder (inputs) \u2192 role+name (buttons/links) \u2192 heading \u2192 text \u2192 id\n */\n\nexport interface PlaywrightLocator {\n  key: string;           // camelCase property name for page object\n  expression: string;    // e.g. \"page.getByRole('button', { name: 'Submit' })\"\n  elementType: string;   // button | input | link | heading | text | element\n  confidence: 'high' | 'medium' | 'low';\n  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';\n  description: string;   // human-readable label\n}\n\n/** Parse the notes JSON string stored on Element records. */\nfunction parseNotes(notes: unknown): Record<string, string> {\n  if (!notes) return {};\n  try {\n    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;\n    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;\n  } catch {}\n  return {};\n}\n\n/** Map crawler role strings to Playwright role values. */\nfunction mapRole(role?: string | null): string | null {\n  if (!role) return null;\n  const r = role.toLowerCase().trim();\n  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';\n  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';\n  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';\n  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';\n  if (r === 'checkbox') return 'checkbox';\n  if (r === 'radio') return 'radio';\n  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';\n  if (r === 'img' || r === 'image') return 'img';\n  if (r === 'nav' || r === 'navigation') return 'navigation';\n  return r;\n}\n\n/** Convert \"Submit Form Button\" \u2192 \"submitFormButton\" */\nfunction toKey(text: string): string {\n  return text\n    .replace(/[^a-zA-Z0-9\\s]/g, ' ')\n    .replace(/\\s+(.)/g, (_, c) => c.toUpperCase())\n    .replace(/^(.)/, (_, c) => c.toLowerCase())\n    .replace(/\\s/g, '')\n    .slice(0, 40)\n    || 'element';\n}\n\n/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */\nexport function elementToPlaywrightLocator(element: {\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n}): PlaywrightLocator | null {\n  const attrs = parseNotes(element.notes);\n  const testId = attrs['data-testid'];\n  const ariaLabel = attrs['aria-label'] || element.meaning;\n  const id = attrs['id'];\n  const placeholder = attrs['placeholder'];\n  const name = attrs['name'];\n  const text = (element.text ?? '').trim();\n  const meaning = (element.meaning ?? '').trim();\n  const role = mapRole(element.role);\n\n  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';\n  const key = toKey(label);\n\n  // Priority 1: data-testid\n  if (testId) {\n    return {\n      key,\n      expression: `page.getByTestId(${JSON.stringify(testId)})`,\n      elementType: role ?? 'element',\n      confidence: 'high',\n      source: 'testid',\n      description: meaning || testId,\n    };\n  }\n\n  // Priority 2: input/combobox/textarea with label \u2192 getByLabel; with placeholder \u2192 getByPlaceholder\n  if (role === 'textbox' || role === 'combobox') {\n    if (ariaLabel) {\n      return {\n        key,\n        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'label',\n        description: ariaLabel,\n      };\n    }\n    if (placeholder) {\n      return {\n        key,\n        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'placeholder',\n        description: placeholder,\n      };\n    }\n  }\n\n  // Priority 3: checkbox/radio \u2192 getByLabel\n  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {\n    return {\n      key,\n      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n      elementType: role,\n      confidence: 'high',\n      source: 'label',\n      description: ariaLabel,\n    };\n  }\n\n  // Priority 4: button/link with meaningful name \u2192 getByRole\n  if ((role === 'button' || role === 'link') && meaning) {\n    return {\n      key,\n      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,\n      elementType: role,\n      confidence: 'high',\n      source: 'role-name',\n      description: meaning,\n    };\n  }\n\n  // Priority 5: heading \u2192 getByRole heading\n  if (role === 'heading' && (meaning || text)) {\n    const name2 = meaning || text;\n    return {\n      key,\n      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,\n      elementType: 'heading',\n      confidence: 'medium',\n      source: 'heading',\n      description: name2,\n    };\n  }\n\n  // Priority 6: existing CSS selector from crawler\n  if (element.selector && element.selector.length > 0) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify(element.selector)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || text || element.selector,\n    };\n  }\n\n  // Priority 7: id from attrs\n  if (id) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify('#' + id)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || id,\n    };\n  }\n\n  // Priority 8: fallback to text content\n  if (meaning || text) {\n    const t = meaning || text;\n    return {\n      key,\n      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,\n      elementType: role ?? 'text',\n      confidence: 'low',\n      source: 'text',\n      description: t,\n    };\n  }\n\n  return null;\n}\n\n/**\n * Convert an array of DB Element records to deduplicated Playwright locators.\n * Deduplicates by key \u2014 higher confidence wins.\n */\nexport function elementsToPlaywrightLocators(elements: Array<{\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n  isInteractive?: boolean | null;\n}>): PlaywrightLocator[] {\n  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };\n  const byKey = new Map<string, PlaywrightLocator>();\n\n  for (const el of elements) {\n    const loc = elementToPlaywrightLocator(el);\n    if (!loc) continue;\n    const existing = byKey.get(loc.key);\n    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {\n      byKey.set(loc.key, loc);\n    }\n  }\n\n  return Array.from(byKey.values());\n}\n\n/**\n * Format locators as a block for injection into LLM script generation prompts.\n * Returns empty string if no locators.\n */\nexport function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {\n  if (locators.length === 0) return '';\n  const lines = locators.map(l =>\n    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`\n  );\n  return [\n    'EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):',\n    ...lines,\n  ].join('\\n');\n}\n"],
+  "mappings": "AAeA,SAAS,WAAW,OAAwC;AAC1D,MAAI,CAAC,MAAO,QAAO,CAAC;AACpB,MAAI;AACF,UAAM,SAAS,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;AAC/D,QAAI,UAAU,OAAO,WAAW,SAAU,QAAO;AAAA,EACnD,QAAQ;AAAA,EAAC;AACT,SAAO,CAAC;AACV;AAGA,SAAS,QAAQ,MAAqC;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,IAAI,KAAK,YAAY,EAAE,KAAK;AAClC,MAAI,MAAM,YAAY,MAAM,YAAY,MAAM,QAAS,QAAO;AAC9D,MAAI,MAAM,UAAU,MAAM,OAAO,MAAM,SAAU,QAAO;AACxD,MAAI,MAAM,aAAa,MAAM,WAAW,MAAM,UAAU,MAAM,WAAW,MAAM,cAAc,MAAM,SAAS,MAAM,YAAY,MAAM,SAAU,QAAO;AACrJ,MAAI,MAAM,cAAc,MAAM,YAAY,MAAM,WAAY,QAAO;AACnE,MAAI,MAAM,WAAY,QAAO;AAC7B,MAAI,MAAM,QAAS,QAAO;AAC1B,MAAI,MAAM,aAAa,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,KAAM,QAAO;AAChH,MAAI,MAAM,SAAS,MAAM,QAAS,QAAO;AACzC,MAAI,MAAM,SAAS,MAAM,aAAc,QAAO;AAC9C,SAAO;AACT;AAGA,SAAS,MAAM,MAAsB;AACnC,SAAO,KACJ,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,WAAW,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EAC5C,QAAQ,QAAQ,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EACzC,QAAQ,OAAO,EAAE,EACjB,MAAM,GAAG,EAAE,KACT;AACP;AAGO,SAAS,2BAA2B,SAMd;AAC3B,QAAM,QAAQ,WAAW,QAAQ,KAAK;AACtC,QAAM,SAAS,MAAM,aAAa;AAClC,QAAM,YAAY,MAAM,YAAY,KAAK,QAAQ;AACjD,QAAM,KAAK,MAAM,IAAI;AACrB,QAAM,cAAc,MAAM,aAAa;AACvC,QAAM,OAAO,MAAM,MAAM;AACzB,QAAM,QAAQ,QAAQ,QAAQ,IAAI,KAAK;AACvC,QAAM,WAAW,QAAQ,WAAW,IAAI,KAAK;AAC7C,QAAM,OAAO,QAAQ,QAAQ,IAAI;AAEjC,QAAM,QAAQ,WAAW,QAAQ,UAAU,aAAa,eAAe,MAAM,QAAQ;AACrF,QAAM,MAAM,MAAM,KAAK;AAGvB,MAAI,QAAQ;AACV,WAAO;AAAA,MACL;AAAA,MACA,YAAY,oBAAoB,KAAK,UAAU,MAAM,CAAC;AAAA,MACtD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,SAAS,aAAa,SAAS,YAAY;AAC7C,QAAI,WAAW;AACb,aAAO;AAAA,QACL;AAAA,QACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,QACxD,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AACA,QAAI,aAAa;AACf,aAAO;AAAA,QACL;AAAA,QACA,YAAY,yBAAyB,KAAK,UAAU,WAAW,CAAC;AAAA,QAChE,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAGA,OAAK,SAAS,cAAc,SAAS,YAAY,WAAW;AAC1D,WAAO;AAAA,MACL;AAAA,MACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,MACxD,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,OAAK,SAAS,YAAY,SAAS,WAAW,SAAS;AACrD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,IAAI,CAAC,aAAa,KAAK,UAAU,OAAO,CAAC;AAAA,MACtF,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,SAAS,cAAc,WAAW,OAAO;AAC3C,UAAM,QAAQ,WAAW;AACzB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,qCAAqC,KAAK,UAAU,KAAK,CAAC;AAAA,MACtE,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,QAAQ,QAAQ,CAAC;AAAA,MAC5D,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW,QAAQ,QAAQ;AAAA,IAC1C;AAAA,EACF;AAGA,MAAI,IAAI;AACN,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,MAAM,EAAE,CAAC;AAAA,MACpD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,CAAC,CAAC;AAAA,MAC/C,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAEA,SAAO;AACT;AAMO,SAAS,6BAA6B,UAOpB;AACvB,QAAM,kBAAkB,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,EAAE;AACrD,QAAM,QAAQ,oBAAI,IAA+B;AAEjD,aAAW,MAAM,UAAU;AACzB,UAAM,MAAM,2BAA2B,EAAE;AACzC,QAAI,CAAC,IAAK;AACV,UAAM,WAAW,MAAM,IAAI,IAAI,GAAG;AAClC,QAAI,CAAC,YAAY,gBAAgB,IAAI,UAAU,IAAI,gBAAgB,SAAS,UAAU,GAAG;AACvF,YAAM,IAAI,IAAI,KAAK,GAAG;AAAA,IACxB;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,MAAM,OAAO,CAAC;AAClC;AAMO,SAAS,wBAAwB,UAAuC;AAC7E,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,QAAQ,SAAS;AAAA,IAAI,OACzB,KAAK,EAAE,GAAG,KAAK,EAAE,UAAU,QAAQ,EAAE,WAAW,KAAK,EAAE,UAAU;AAAA,EACnE;AACA,SAAO;AAAA,IACL;AAAA,IACA,GAAG;AAAA,EACL,EAAE,KAAK,IAAI;AACb;",
+  "names": []
+}

package/dist/sitemap.js

@@ -60,9 +60,11 @@ async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
   if (!parsed) return [];
   const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
   const collected = [...pageUrls];
-  for (const subUrl of subSitemapUrls) {
-    if (collected.length >= MAX_URLS) break;
-    collected.push(...await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher));
+  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+    );
+    for (const sub of subResults) collected.push(...sub);
   }
   return [...new Set(collected)].slice(0, MAX_URLS);
 }

package/dist/sitemap.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/sitemap.ts"],
-  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  for (const subUrl of subSitemapUrls) {\n    if (collected.length >= MAX_URLS) break;\n    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
-  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,aAAW,UAAU,gBAAgB;AACnC,QAAI,UAAU,UAAU,SAAU;AAClC,cAAU,KAAK,GAAI,MAAM,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAE;AAAA,EACnF;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
+  "sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n  if (x == null) return [];\n  return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n  return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n  try {\n    const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n  const rawPageUrls: string[] = [];\n  let subSitemapUrls: string[] = [];\n\n  if (parsed?.sitemapindex) {\n    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n      .map((s: any) => s?.loc)\n      .filter((loc: any): loc is string => typeof loc === 'string')\n      .slice(0, MAX_SUB_SITEMAPS);\n  }\n\n  if (parsed?.urlset) {\n    rawPageUrls.push(\n      ...toArray(parsed.urlset.url)\n        .map((u: any) => u?.loc)\n        .filter((loc: any): loc is string => typeof loc === 'string'),\n    );\n  }\n\n  // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n  // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n  // living on a tenant's internal/VPN-only staging server is a legitimate\n  // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n  try {\n    assertCrawlUrlSafe(url);\n  } catch {\n    return null;\n  }\n  try {\n    // Redirects followed normally here (unlike the webhook-dispatch.ts\n    // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n    // sitemap is completely ordinary, and Playwright's actual page\n    // navigation later in the crawl already follows redirects to whatever\n    // this same site sends it to; a sitemap fetch isn't a more sensitive\n    // operation than that.\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (!res.ok) return null;\n    const text = await res.text();\n    return new XMLParser().parse(text);\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n  if (!parsed) return [];\n\n  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n  const collected = [...pageUrls];\n\n  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {\n    const subResults = await Promise.all(\n      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))\n    );\n    for (const sub of subResults) collected.push(...sub);\n  }\n\n  return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n  let origin: string;\n  try {\n    origin = new URL(appUrl).origin;\n  } catch {\n    return [];\n  }\n  const target = sitemapUrl || `${origin}/sitemap.xml`;\n  try {\n    return await fetchSitemapUrls(target, origin);\n  } catch (err) {\n    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n    return [];\n  }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n  if (!proxyUrl || proxyUrl === 'auto') return undefined;\n  try {\n    const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n    if (!m) return new ProxyAgent(proxyUrl);\n    const [, scheme, user, pass, hostPort] = m;\n    const uri = `${scheme}://${hostPort}`;\n    if (user) {\n      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n      return new ProxyAgent({ uri, token });\n    }\n    return new ProxyAgent({ uri });\n  } catch {\n    return undefined;\n  }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n  const out: string[] = [];\n  for (const line of robotsTxt.split(/\\r?\\n/)) {\n    const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n    if (m && sameSite(m[1], origin)) out.push(m[1]);\n  }\n  return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n  url: string;\n  source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n  appUrl: string,\n  opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n  let origin: string;\n  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n  // 1. robots.txt \u2192 declared sitemaps\n  let robotsSitemaps: string[] = [];\n  try {\n    assertCrawlUrlSafe(`${origin}/robots.txt`);\n    const res = await fetch(`${origin}/robots.txt`, {\n      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n      ...(dispatcher ? { dispatcher } : {}),\n    });\n    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n  } catch { /* best-effort */ }\n\n  // 2. sitemap targets: explicit override + robots-declared + conventional default\n  const robotsSet = new Set(robotsSitemaps);\n  const sitemapTargets = [...new Set([\n    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n    ...robotsSitemaps,\n    `${origin}/sitemap.xml`,\n  ])];\n\n  const seen = new Map<string, UrlCandidate>();\n  let sitemapFound = false;\n  for (const sm of sitemapTargets) {\n    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n    if (urls.length) sitemapFound = true;\n    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n  }\n\n  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
+  "mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,MAAI,eAAe,SAAS,KAAK,UAAU,SAAS,UAAU;AAC5D,UAAM,aAAa,MAAM,QAAQ;AAAA,MAC/B,eAAe,MAAM,GAAG,EAAE,EAAE,IAAI,YAAU,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAC;AAAA,IACnG;AACA,eAAW,OAAO,WAAY,WAAU,KAAK,GAAG,GAAG;AAAA,EACrD;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
   "names": []
 }

package/dist/url-pattern-completer.js

@@ -47,6 +47,7 @@ Respond with ONLY valid JSON, no explanation:
     const response = await client.messages.create({
       model: llmCfg.model ?? "claude-haiku-4-5-20251001",
       max_tokens: 1024,
+      temperature: 0,
       // @ts-ignore: cache_control supported in beta
       system: [{ type: "text", text: "You are a URL pattern analyst. Always respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
       messages: [{ role: "user", content: prompt }]

package/dist/url-pattern-completer.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/url-pattern-completer.ts"],
-  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
-  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
+  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      temperature: 0,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
+  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA,MACZ,aAAa;AAAA;AAAA,MAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.6",
+  "version": "1.1.8",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/index.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';

package/src/locator-extractor.ts

@@ -0,0 +1,226 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+
+export interface PlaywrightLocator {
+  key: string;           // camelCase property name for page object
+  expression: string;    // e.g. "page.getByRole('button', { name: 'Submit' })"
+  elementType: string;   // button | input | link | heading | text | element
+  confidence: 'high' | 'medium' | 'low';
+  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+  description: string;   // human-readable label
+}
+
+/** Parse the notes JSON string stored on Element records. */
+function parseNotes(notes: unknown): Record<string, string> {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;
+  } catch {}
+  return {};
+}
+
+/** Map crawler role strings to Playwright role values. */
+function mapRole(role?: string | null): string | null {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';
+  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';
+  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';
+  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';
+  if (r === 'checkbox') return 'checkbox';
+  if (r === 'radio') return 'radio';
+  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';
+  if (r === 'img' || r === 'image') return 'img';
+  if (r === 'nav' || r === 'navigation') return 'navigation';
+  return r;
+}
+
+/** Convert "Submit Form Button" → "submitFormButton" */
+function toKey(text: string): string {
+  return text
+    .replace(/[^a-zA-Z0-9\s]/g, ' ')
+    .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
+    .replace(/^(.)/, (_, c) => c.toLowerCase())
+    .replace(/\s/g, '')
+    .slice(0, 40)
+    || 'element';
+}
+
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export function elementToPlaywrightLocator(element: {
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+}): PlaywrightLocator | null {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs['data-testid'];
+  const ariaLabel = attrs['aria-label'] || element.meaning;
+  const id = attrs['id'];
+  const placeholder = attrs['placeholder'];
+  const name = attrs['name'];
+  const text = (element.text ?? '').trim();
+  const meaning = (element.meaning ?? '').trim();
+  const role = mapRole(element.role);
+
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
+  const key = toKey(label);
+
+  // Priority 1: data-testid
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? 'element',
+      confidence: 'high',
+      source: 'testid',
+      description: meaning || testId,
+    };
+  }
+
+  // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
+  if (role === 'textbox' || role === 'combobox') {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'label',
+        description: ariaLabel,
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'placeholder',
+        description: placeholder,
+      };
+    }
+  }
+
+  // Priority 3: checkbox/radio → getByLabel
+  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'label',
+      description: ariaLabel,
+    };
+  }
+
+  // Priority 4: button/link with meaningful name → getByRole
+  if ((role === 'button' || role === 'link') && meaning) {
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      elementType: role,
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+
+  // Priority 5: heading → getByRole heading
+  if (role === 'heading' && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: 'heading',
+      confidence: 'medium',
+      source: 'heading',
+      description: name2,
+    };
+  }
+
+  // Priority 6: existing CSS selector from crawler
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || text || element.selector,
+    };
+  }
+
+  // Priority 7: id from attrs
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify('#' + id)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || id,
+    };
+  }
+
+  // Priority 8: fallback to text content
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? 'text',
+      confidence: 'low',
+      source: 'text',
+      description: t,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export function elementsToPlaywrightLocators(elements: Array<{
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+  isInteractive?: boolean | null;
+}>): PlaywrightLocator[] {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = new Map<string, PlaywrightLocator>();
+
+  for (const el of elements) {
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+
+  return Array.from(byKey.values());
+}
+
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {
+  if (locators.length === 0) return '';
+  const lines = locators.map(l =>
+    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
+    ...lines,
+  ].join('\n');
+}

package/types/index.d.ts

@@ -35,3 +35,5 @@ export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemet
 export type { Span, FinishedSpan } from './telemetry.js';
 export { analyzeFailure } from './failure-analysis-agent.js';
 export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';