@detiq/crawler 1.1.13 → 1.1.15

package/dist/filter-pagination-crawler.js

@@ -0,0 +1,299 @@
+async function waitForSettle(page, ms = 1500) {
+  await Promise.race([
+    page.waitForLoadState("networkidle").catch(() => void 0),
+    new Promise((r) => setTimeout(r, ms))
+  ]);
+}
+async function captureFirstRow(page) {
+  try {
+    return await page.evaluate(() => {
+      const row = document.querySelector("tbody tr");
+      if (!row) return [];
+      return Array.from(row.querySelectorAll("td")).map((td) => td.innerText.trim());
+    });
+  } catch {
+    return [];
+  }
+}
+async function readResultCount(page) {
+  try {
+    const text = await page.evaluate(() => document.body.innerText);
+    const match = text.match(/(\d[\d,]*)\s*(result|item|entr|record)/i);
+    if (!match) return null;
+    return parseInt(match[1].replace(/,/g, ""), 10);
+  } catch {
+    return null;
+  }
+}
+async function discoverFilters(page) {
+  const filters = [];
+  try {
+    const selectHandles = await page.locator('select:not([role="dialog"] select)').all();
+    for (const handle of selectHandles.slice(0, 10)) {
+      try {
+        const id = await handle.getAttribute("id").catch(() => null);
+        const name = await handle.getAttribute("name").catch(() => null);
+        const locatorHint = id ? `#${id}` : name ? `select[name="${name}"]` : "select";
+        let label = "";
+        if (id) {
+          label = await page.locator(`label[for="${id}"]`).innerText().catch(() => "");
+        }
+        if (!label) {
+          label = await handle.evaluate((el) => {
+            const lbl = el.closest("label") ?? el.previousElementSibling;
+            return lbl?.innerText?.trim() ?? "";
+          }).catch(() => "");
+        }
+        if (!label) label = name ?? id ?? "filter";
+        const rawOptions = await handle.evaluate(
+          (el) => Array.from(el.options).map((o) => ({ value: o.value, label: o.text.trim() }))
+        ).catch(() => []);
+        const options = rawOptions.filter(
+          (o) => !(/^all$|^any$|^select|^--/i.test(o.label) || o.value === "")
+        );
+        if (options.length === 0) continue;
+        filters.push({ kind: "select", label, locatorHint, options });
+        if (filters.length >= 10) break;
+      } catch {
+      }
+    }
+  } catch {
+  }
+  if (filters.length < 10) {
+    try {
+      const searchHandles = await page.locator(
+        'input[type="search"]:not([role="dialog"] input), input[placeholder*="search" i]:not([role="dialog"] input)'
+      ).all();
+      for (const handle of searchHandles.slice(0, 10 - filters.length)) {
+        try {
+          const id = await handle.getAttribute("id").catch(() => null);
+          const placeholder = await handle.getAttribute("placeholder").catch(() => null);
+          const locatorHint = id ? `#${id}` : placeholder ? `input[placeholder="${placeholder}"]` : 'input[type="search"]';
+          let label = "";
+          if (id) {
+            label = await page.locator(`label[for="${id}"]`).innerText().catch(() => "");
+          }
+          if (!label) label = placeholder ?? "search";
+          filters.push({ kind: "search", label, locatorHint, options: [] });
+          if (filters.length >= 10) break;
+        } catch {
+        }
+      }
+    } catch {
+    }
+  }
+  return filters;
+}
+async function applyFilterAndCapture(page, filter, option) {
+  const state = {
+    filterLabel: filter.label,
+    filterValue: option.value,
+    resultCount: null,
+    firstRowValues: []
+  };
+  try {
+    if (filter.kind === "select") {
+      await page.locator(filter.locatorHint).selectOption(option.value);
+    } else if (filter.kind === "search") {
+      await page.locator(filter.locatorHint).fill(option.value);
+      await page.locator(filter.locatorHint).press("Enter");
+    }
+    await waitForSettle(page, 1500);
+    state.resultCount = await readResultCount(page);
+    state.firstRowValues = await captureFirstRow(page);
+  } catch {
+  }
+  try {
+    if (filter.kind === "select") {
+      const firstOption = await page.locator(filter.locatorHint).evaluate(
+        (el) => el.options[0]?.value ?? ""
+      );
+      await page.locator(filter.locatorHint).selectOption(firstOption);
+    } else if (filter.kind === "search") {
+      await page.locator(filter.locatorHint).fill("");
+      await page.locator(filter.locatorHint).press("Enter");
+    }
+    await waitForSettle(page, 800);
+  } catch {
+  }
+  return state;
+}
+async function crawlFilterStates(page, filters) {
+  const states = [];
+  let totalStates = 0;
+  for (const filter of filters) {
+    if (totalStates >= 15) break;
+    const { options } = filter;
+    if (options.length === 0) continue;
+    const indices = /* @__PURE__ */ new Set([
+      0,
+      Math.floor((options.length - 1) / 2),
+      options.length - 1
+    ]);
+    const picked = Array.from(indices).slice(0, 3);
+    for (const idx of picked) {
+      if (totalStates >= 15) break;
+      const option = options[idx];
+      try {
+        const s = await applyFilterAndCapture(page, filter, option);
+        states.push(s);
+      } catch {
+      }
+      totalStates++;
+    }
+  }
+  return states;
+}
+async function detectPaginationKind(page) {
+  try {
+    const numberedCount = await page.locator('[aria-label*="page" i], .pagination button, [role="navigation"] button').count().catch(() => 0);
+    if (numberedCount > 1) return "numbered";
+    const loadMore = await page.locator('button:has-text("Load more"), button:has-text("Show more"), a:has-text("Load more")').count().catch(() => 0);
+    if (loadMore > 0) return "load-more";
+    const isInfinite = await page.evaluate(() => {
+      return document.documentElement.scrollHeight > window.innerHeight * 2;
+    }).catch(() => false);
+    if (isInfinite) return "infinite-scroll";
+  } catch {
+  }
+  return "none";
+}
+async function crawlNumberedPagination(page) {
+  const pages = [];
+  const pageSizes = [];
+  try {
+    const sizeSelect = page.locator('select[name*="size" i], select[name*="per" i], select[aria-label*="per page" i]');
+    const sizeCount = await sizeSelect.count();
+    if (sizeCount > 0) {
+      const rawSizes = await sizeSelect.first().evaluate(
+        (el) => Array.from(el.options).map((o) => parseInt(o.value, 10)).filter((n) => !isNaN(n))
+      );
+      pageSizes.push(...rawSizes);
+    }
+  } catch {
+  }
+  let totalPages;
+  try {
+    totalPages = await page.evaluate(() => {
+      const buttons = Array.from(
+        document.querySelectorAll('[aria-label*="page" i], .pagination button')
+      );
+      const nums = buttons.map((b) => parseInt(b.innerText.trim(), 10)).filter((n) => !isNaN(n));
+      return nums.length > 0 ? Math.max(...nums) : void 0;
+    });
+  } catch {
+  }
+  pages.push({ pageNum: 1, rowValues: await captureFirstRow(page) });
+  for (let pageNum = 2; pageNum <= 20; pageNum++) {
+    try {
+      const nextBtn = page.locator(
+        'button[aria-label*="next" i], a[aria-label*="next" i], [role="navigation"] button:last-child'
+      ).first();
+      const isDisabled = await nextBtn.isDisabled().catch(() => true);
+      if (isDisabled) break;
+      await nextBtn.click();
+      await waitForSettle(page, 2e3);
+      const rowValues = await captureFirstRow(page);
+      pages.push({ pageNum, rowValues });
+    } catch {
+      break;
+    }
+  }
+  return {
+    kind: "numbered",
+    totalPages,
+    pageSizes: pageSizes.length > 0 ? pageSizes : void 0,
+    pages
+  };
+}
+async function captureInfiniteScroll(page) {
+  const pages = [];
+  let prevRowCount = -1;
+  let iteration = 0;
+  while (iteration < 10) {
+    try {
+      const rowCount = await page.evaluate(
+        () => document.querySelectorAll("tbody tr").length
+      ).catch(() => 0);
+      if (rowCount === prevRowCount) break;
+      prevRowCount = rowCount;
+      pages.push({ pageNum: iteration + 1, rowValues: await captureFirstRow(page) });
+      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+      await new Promise((r) => setTimeout(r, 800));
+    } catch {
+      break;
+    }
+    iteration++;
+  }
+  const totalItems = await page.evaluate(
+    () => document.querySelectorAll("tbody tr").length
+  ).catch(() => void 0);
+  return {
+    kind: "infinite-scroll",
+    totalItems: typeof totalItems === "number" ? totalItems : void 0,
+    pages
+  };
+}
+async function captureSortColumns(page) {
+  const results = [];
+  try {
+    const sortable = await page.locator("[aria-sort]").all();
+    for (const th of sortable.slice(0, 5)) {
+      try {
+        const header = await th.innerText().catch(() => "");
+        await th.click();
+        await waitForSettle(page, 800);
+        const ariaSortAfterClick = await th.getAttribute("aria-sort").catch(() => null);
+        results.push({ header: header.trim(), ariaSortAfterClick });
+        await th.click().catch(() => void 0);
+        await waitForSettle(page, 600);
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return results;
+}
+async function crawlFiltersAndPagination(page) {
+  let filters = [];
+  try {
+    filters = await discoverFilters(page);
+  } catch {
+  }
+  let filterStates = [];
+  try {
+    filterStates = await crawlFilterStates(page, filters);
+  } catch {
+  }
+  let pagination = { kind: "none", pages: [] };
+  try {
+    const kind = await detectPaginationKind(page);
+    if (kind === "numbered") {
+      pagination = await crawlNumberedPagination(page);
+    } else if (kind === "load-more") {
+      pagination = await captureInfiniteScroll(page);
+      pagination = { ...pagination, kind: "load-more" };
+    } else if (kind === "infinite-scroll") {
+      pagination = await captureInfiniteScroll(page);
+    } else {
+      const rowValues = await captureFirstRow(page);
+      pagination = { kind: "none", pages: rowValues.length > 0 ? [{ pageNum: 1, rowValues }] : [] };
+    }
+  } catch {
+  }
+  let sortColumns = [];
+  try {
+    sortColumns = await captureSortColumns(page);
+  } catch {
+  }
+  return {
+    filters,
+    filterStates,
+    pagination,
+    sortColumns
+  };
+}
+export {
+  crawlFiltersAndPagination
+};
+//# sourceMappingURL=filter-pagination-crawler.js.map

package/dist/filter-pagination-crawler.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/filter-pagination-crawler.ts"],
+  "sourcesContent": ["/**\n * Filter state crawling and pagination capture.\n *\n * Discovers filter controls (select, checkbox-group, radio-group, search),\n * applies each meaningful option and records result counts and first-row\n * values, then detects and walks pagination (numbered, load-more, infinite\n * scroll). Also captures sort column behaviour.\n *\n * All per-filter / per-page errors are caught and skipped so one bad control\n * never aborts the full crawl.\n */\n\nimport type { Page } from 'playwright';\n\n// ---------------------------------------------------------------------------\n// Public interfaces\n// ---------------------------------------------------------------------------\n\nexport interface FilterOption {\n  value: string;\n  label: string;\n}\n\nexport interface FilterDef {\n  kind: 'select' | 'checkbox-group' | 'radio-group' | 'search';\n  label: string;\n  locatorHint: string;\n  options: FilterOption[];\n}\n\nexport interface FilterState {\n  filterLabel: string;\n  filterValue: string;\n  resultCount: number | null;\n  firstRowValues: string[];\n}\n\nexport interface PaginationCapture {\n  kind: 'numbered' | 'load-more' | 'infinite-scroll' | 'none';\n  totalPages?: number;\n  totalItems?: number;\n  pageSizes?: number[];\n  pages: Array<{ pageNum: number; rowValues: string[] }>;\n}\n\nexport interface FilterPaginationResult {\n  filters: FilterDef[];\n  filterStates: FilterState[];\n  pagination: PaginationCapture;\n  sortColumns: Array<{ header: string; ariaSortAfterClick: string | null }>;\n  searchCapture?: { noResultsText: string | null; sampleSuggestions: string[] };\n}\n\n// ---------------------------------------------------------------------------\n// Internal helpers\n// ---------------------------------------------------------------------------\n\n/** Pause for up to `ms` ms or until network is idle, whichever comes first. */\nasync function waitForSettle(page: Page, ms = 1500): Promise<void> {\n  await Promise.race([\n    page.waitForLoadState('networkidle').catch(() => undefined),\n    new Promise<void>(r => setTimeout(r, ms)),\n  ]);\n}\n\n/** Extract visible cell text from the first tbody row, if any. */\nasync function captureFirstRow(page: Page): Promise<string[]> {\n  try {\n    return await page.evaluate(() => {\n      const row = document.querySelector('tbody tr');\n      if (!row) return [];\n      return Array.from(row.querySelectorAll('td')).map(td => td.innerText.trim());\n    });\n  } catch {\n    return [];\n  }\n}\n\n/** Read a result-count hint from visible page text using common patterns. */\nasync function readResultCount(page: Page): Promise<number | null> {\n  try {\n    const text = await page.evaluate(() => document.body.innerText);\n    const match = text.match(/(\\d[\\d,]*)\\s*(result|item|entr|record)/i);\n    if (!match) return null;\n    return parseInt(match[1].replace(/,/g, ''), 10);\n  } catch {\n    return null;\n  }\n}\n\n// ---------------------------------------------------------------------------\n// discoverFilters\n// ---------------------------------------------------------------------------\n\nasync function discoverFilters(page: Page): Promise<FilterDef[]> {\n  const filters: FilterDef[] = [];\n\n  // --- native <select> elements outside dialogs ---------------------------------\n  try {\n    const selectHandles = await page.locator('select:not([role=\"dialog\"] select)').all();\n    for (const handle of selectHandles.slice(0, 10)) {\n      try {\n        const id = await handle.getAttribute('id').catch(() => null);\n        const name = await handle.getAttribute('name').catch(() => null);\n        const locatorHint = id ? `#${id}` : name ? `select[name=\"${name}\"]` : 'select';\n\n        // Find associated label\n        let label = '';\n        if (id) {\n          label = await page.locator(`label[for=\"${id}\"]`).innerText().catch(() => '');\n        }\n        if (!label) {\n          label = await handle.evaluate((el: HTMLSelectElement) => {\n            const lbl = el.closest('label') ?? el.previousElementSibling;\n            return (lbl as HTMLElement | null)?.innerText?.trim() ?? '';\n          }).catch(() => '');\n        }\n        if (!label) label = name ?? id ?? 'filter';\n\n        // Collect options, skipping default/placeholder entries\n        const rawOptions: FilterOption[] = await handle.evaluate((el: HTMLSelectElement) =>\n          Array.from(el.options).map(o => ({ value: o.value, label: o.text.trim() })),\n        ).catch(() => []);\n\n        const options = rawOptions.filter(\n          o => !(/^all$|^any$|^select|^--/i.test(o.label) || o.value === ''),\n        );\n\n        if (options.length === 0) continue;\n\n        filters.push({ kind: 'select', label, locatorHint, options });\n        if (filters.length >= 10) break;\n      } catch {\n        // skip this select\n      }\n    }\n  } catch {\n    // page.locator failed\n  }\n\n  // --- search inputs ------------------------------------------------------------\n  if (filters.length < 10) {\n    try {\n      const searchHandles = await page\n        .locator(\n          'input[type=\"search\"]:not([role=\"dialog\"] input), ' +\n          'input[placeholder*=\"search\" i]:not([role=\"dialog\"] input)',\n        )\n        .all();\n\n      for (const handle of searchHandles.slice(0, 10 - filters.length)) {\n        try {\n          const id = await handle.getAttribute('id').catch(() => null);\n          const placeholder = await handle.getAttribute('placeholder').catch(() => null);\n          const locatorHint = id ? `#${id}` : placeholder ? `input[placeholder=\"${placeholder}\"]` : 'input[type=\"search\"]';\n\n          let label = '';\n          if (id) {\n            label = await page.locator(`label[for=\"${id}\"]`).innerText().catch(() => '');\n          }\n          if (!label) label = placeholder ?? 'search';\n\n          filters.push({ kind: 'search', label, locatorHint, options: [] });\n          if (filters.length >= 10) break;\n        } catch {\n          // skip\n        }\n      }\n    } catch {\n      // locator failed\n    }\n  }\n\n  return filters;\n}\n\n// ---------------------------------------------------------------------------\n// applyFilterAndCapture\n// ---------------------------------------------------------------------------\n\nasync function applyFilterAndCapture(\n  page: Page,\n  filter: FilterDef,\n  option: FilterOption,\n): Promise<FilterState> {\n  const state: FilterState = {\n    filterLabel: filter.label,\n    filterValue: option.value,\n    resultCount: null,\n    firstRowValues: [],\n  };\n\n  try {\n    if (filter.kind === 'select') {\n      await page.locator(filter.locatorHint).selectOption(option.value);\n    } else if (filter.kind === 'search') {\n      await page.locator(filter.locatorHint).fill(option.value);\n      await page.locator(filter.locatorHint).press('Enter');\n    }\n\n    await waitForSettle(page, 1500);\n\n    state.resultCount = await readResultCount(page);\n    state.firstRowValues = await captureFirstRow(page);\n  } catch {\n    // non-fatal: return partial state\n  }\n\n  // Reset filter\n  try {\n    if (filter.kind === 'select') {\n      const firstOption = await page.locator(filter.locatorHint).evaluate(\n        (el: HTMLSelectElement) => el.options[0]?.value ?? '',\n      );\n      await page.locator(filter.locatorHint).selectOption(firstOption);\n    } else if (filter.kind === 'search') {\n      await page.locator(filter.locatorHint).fill('');\n      await page.locator(filter.locatorHint).press('Enter');\n    }\n    await waitForSettle(page, 800);\n  } catch {\n    // ignore reset errors\n  }\n\n  return state;\n}\n\n// ---------------------------------------------------------------------------\n// crawlFilterStates\n// ---------------------------------------------------------------------------\n\nasync function crawlFilterStates(page: Page, filters: FilterDef[]): Promise<FilterState[]> {\n  const states: FilterState[] = [];\n  let totalStates = 0;\n\n  for (const filter of filters) {\n    if (totalStates >= 15) break;\n\n    const { options } = filter;\n    if (options.length === 0) continue;\n\n    // Pick first, middle, last \u2014 deduplicated, max 3\n    const indices = new Set<number>([\n      0,\n      Math.floor((options.length - 1) / 2),\n      options.length - 1,\n    ]);\n    const picked = Array.from(indices).slice(0, 3);\n\n    for (const idx of picked) {\n      if (totalStates >= 15) break;\n      const option = options[idx];\n      try {\n        const s = await applyFilterAndCapture(page, filter, option);\n        states.push(s);\n      } catch {\n        // skip\n      }\n      totalStates++;\n    }\n  }\n\n  return states;\n}\n\n// ---------------------------------------------------------------------------\n// detectPaginationKind\n// ---------------------------------------------------------------------------\n\nasync function detectPaginationKind(\n  page: Page,\n): Promise<'numbered' | 'load-more' | 'infinite-scroll' | 'none'> {\n  try {\n    // Numbered pagination: ARIA page buttons or .pagination\n    const numberedCount = await page\n      .locator('[aria-label*=\"page\" i], .pagination button, [role=\"navigation\"] button')\n      .count()\n      .catch(() => 0);\n    if (numberedCount > 1) return 'numbered';\n\n    // Load-more button\n    const loadMore = await page\n      .locator('button:has-text(\"Load more\"), button:has-text(\"Show more\"), a:has-text(\"Load more\")')\n      .count()\n      .catch(() => 0);\n    if (loadMore > 0) return 'load-more';\n\n    // Infinite scroll: scrollable content significantly taller than viewport\n    const isInfinite = await page.evaluate(() => {\n      return document.documentElement.scrollHeight > window.innerHeight * 2;\n    }).catch(() => false);\n    if (isInfinite) return 'infinite-scroll';\n  } catch {\n    // fall through\n  }\n\n  return 'none';\n}\n\n// ---------------------------------------------------------------------------\n// crawlNumberedPagination\n// ---------------------------------------------------------------------------\n\nasync function crawlNumberedPagination(page: Page): Promise<PaginationCapture> {\n  const pages: Array<{ pageNum: number; rowValues: string[] }> = [];\n  const pageSizes: number[] = [];\n\n  // Read page size selector if present\n  try {\n    const sizeSelect = page.locator('select[name*=\"size\" i], select[name*=\"per\" i], select[aria-label*=\"per page\" i]');\n    const sizeCount = await sizeSelect.count();\n    if (sizeCount > 0) {\n      const rawSizes = await sizeSelect.first().evaluate((el: HTMLSelectElement) =>\n        Array.from(el.options).map(o => parseInt(o.value, 10)).filter(n => !isNaN(n)),\n      );\n      pageSizes.push(...rawSizes);\n    }\n  } catch {\n    // ignore\n  }\n\n  // Determine total pages from pagination widget\n  let totalPages: number | undefined;\n  try {\n    totalPages = await page.evaluate(() => {\n      const buttons = Array.from(\n        document.querySelectorAll('[aria-label*=\"page\" i], .pagination button'),\n      );\n      const nums = buttons\n        .map(b => parseInt((b as HTMLElement).innerText.trim(), 10))\n        .filter(n => !isNaN(n));\n      return nums.length > 0 ? Math.max(...nums) : undefined;\n    });\n  } catch {\n    // leave undefined\n  }\n\n  // Capture first page immediately\n  pages.push({ pageNum: 1, rowValues: await captureFirstRow(page) });\n\n  // Walk pages by clicking Next\n  for (let pageNum = 2; pageNum <= 20; pageNum++) {\n    try {\n      const nextBtn = page.locator(\n        'button[aria-label*=\"next\" i], a[aria-label*=\"next\" i], [role=\"navigation\"] button:last-child',\n      ).first();\n\n      const isDisabled = await nextBtn.isDisabled().catch(() => true);\n      if (isDisabled) break;\n\n      await nextBtn.click();\n      await waitForSettle(page, 2000);\n\n      const rowValues = await captureFirstRow(page);\n      pages.push({ pageNum, rowValues });\n    } catch {\n      break;\n    }\n  }\n\n  return {\n    kind: 'numbered',\n    totalPages,\n    pageSizes: pageSizes.length > 0 ? pageSizes : undefined,\n    pages,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// captureInfiniteScroll\n// ---------------------------------------------------------------------------\n\nasync function captureInfiniteScroll(page: Page): Promise<PaginationCapture> {\n  const pages: Array<{ pageNum: number; rowValues: string[] }> = [];\n\n  let prevRowCount = -1;\n  let iteration = 0;\n\n  while (iteration < 10) {\n    try {\n      const rowCount = await page.evaluate(() =>\n        document.querySelectorAll('tbody tr').length,\n      ).catch(() => 0);\n\n      if (rowCount === prevRowCount) break; // stable\n      prevRowCount = rowCount;\n\n      // Record current first-row snapshot per scroll step\n      pages.push({ pageNum: iteration + 1, rowValues: await captureFirstRow(page) });\n\n      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));\n      await new Promise<void>(r => setTimeout(r, 800));\n    } catch {\n      break;\n    }\n    iteration++;\n  }\n\n  const totalItems = await page.evaluate(() =>\n    document.querySelectorAll('tbody tr').length,\n  ).catch(() => undefined);\n\n  return {\n    kind: 'infinite-scroll',\n    totalItems: typeof totalItems === 'number' ? totalItems : undefined,\n    pages,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// captureSortColumns\n// ---------------------------------------------------------------------------\n\nasync function captureSortColumns(\n  page: Page,\n): Promise<Array<{ header: string; ariaSortAfterClick: string | null }>> {\n  const results: Array<{ header: string; ariaSortAfterClick: string | null }> = [];\n\n  try {\n    const sortable = await page.locator('[aria-sort]').all();\n    for (const th of sortable.slice(0, 5)) {\n      try {\n        const header = await th.innerText().catch(() => '');\n        await th.click();\n        await waitForSettle(page, 800);\n        const ariaSortAfterClick = await th.getAttribute('aria-sort').catch(() => null);\n        results.push({ header: header.trim(), ariaSortAfterClick });\n\n        // Reset by clicking again (most tables toggle back)\n        await th.click().catch(() => undefined);\n        await waitForSettle(page, 600);\n      } catch {\n        // skip this column\n      }\n    }\n  } catch {\n    // no sortable columns found\n  }\n\n  return results;\n}\n\n// ---------------------------------------------------------------------------\n// crawlFiltersAndPagination \u2014 public orchestrator\n// ---------------------------------------------------------------------------\n\nexport async function crawlFiltersAndPagination(page: Page): Promise<FilterPaginationResult> {\n  // 1. Discover filters\n  let filters: FilterDef[] = [];\n  try {\n    filters = await discoverFilters(page);\n  } catch {\n    // proceed with empty filters\n  }\n\n  // 2. Apply filters and capture states\n  let filterStates: FilterState[] = [];\n  try {\n    filterStates = await crawlFilterStates(page, filters);\n  } catch {\n    // proceed with empty states\n  }\n\n  // 3. Detect and crawl pagination\n  let pagination: PaginationCapture = { kind: 'none', pages: [] };\n  try {\n    const kind = await detectPaginationKind(page);\n\n    if (kind === 'numbered') {\n      pagination = await crawlNumberedPagination(page);\n    } else if (kind === 'load-more') {\n      // Treat load-more similarly to infinite scroll capture\n      pagination = await captureInfiniteScroll(page);\n      pagination = { ...pagination, kind: 'load-more' };\n    } else if (kind === 'infinite-scroll') {\n      pagination = await captureInfiniteScroll(page);\n    } else {\n      // Capture single-page rows\n      const rowValues = await captureFirstRow(page);\n      pagination = { kind: 'none', pages: rowValues.length > 0 ? [{ pageNum: 1, rowValues }] : [] };\n    }\n  } catch {\n    // leave pagination as none\n  }\n\n  // 4. Sort columns\n  let sortColumns: Array<{ header: string; ariaSortAfterClick: string | null }> = [];\n  try {\n    sortColumns = await captureSortColumns(page);\n  } catch {\n    // proceed with empty sort\n  }\n\n  return {\n    filters,\n    filterStates,\n    pagination,\n    sortColumns,\n  };\n}\n"],
+  "mappings": "AA0DA,eAAe,cAAc,MAAY,KAAK,MAAqB;AACjE,QAAM,QAAQ,KAAK;AAAA,IACjB,KAAK,iBAAiB,aAAa,EAAE,MAAM,MAAM,MAAS;AAAA,IAC1D,IAAI,QAAc,OAAK,WAAW,GAAG,EAAE,CAAC;AAAA,EAC1C,CAAC;AACH;AAGA,eAAe,gBAAgB,MAA+B;AAC5D,MAAI;AACF,WAAO,MAAM,KAAK,SAAS,MAAM;AAC/B,YAAM,MAAM,SAAS,cAAc,UAAU;AAC7C,UAAI,CAAC,IAAK,QAAO,CAAC;AAClB,aAAO,MAAM,KAAK,IAAI,iBAAiB,IAAI,CAAC,EAAE,IAAI,QAAM,GAAG,UAAU,KAAK,CAAC;AAAA,IAC7E,CAAC;AAAA,EACH,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAGA,eAAe,gBAAgB,MAAoC;AACjE,MAAI;AACF,UAAM,OAAO,MAAM,KAAK,SAAS,MAAM,SAAS,KAAK,SAAS;AAC9D,UAAM,QAAQ,KAAK,MAAM,yCAAyC;AAClE,QAAI,CAAC,MAAO,QAAO;AACnB,WAAO,SAAS,MAAM,CAAC,EAAE,QAAQ,MAAM,EAAE,GAAG,EAAE;AAAA,EAChD,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAe,gBAAgB,MAAkC;AAC/D,QAAM,UAAuB,CAAC;AAG9B,MAAI;AACF,UAAM,gBAAgB,MAAM,KAAK,QAAQ,oCAAoC,EAAE,IAAI;AACnF,eAAW,UAAU,cAAc,MAAM,GAAG,EAAE,GAAG;AAC/C,UAAI;AACF,cAAM,KAAK,MAAM,OAAO,aAAa,IAAI,EAAE,MAAM,MAAM,IAAI;AAC3D,cAAM,OAAO,MAAM,OAAO,aAAa,MAAM,EAAE,MAAM,MAAM,IAAI;AAC/D,cAAM,cAAc,KAAK,IAAI,EAAE,KAAK,OAAO,gBAAgB,IAAI,OAAO;AAGtE,YAAI,QAAQ;AACZ,YAAI,IAAI;AACN,kBAAQ,MAAM,KAAK,QAAQ,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,MAAM,EAAE;AAAA,QAC7E;AACA,YAAI,CAAC,OAAO;AACV,kBAAQ,MAAM,OAAO,SAAS,CAAC,OAA0B;AACvD,kBAAM,MAAM,GAAG,QAAQ,OAAO,KAAK,GAAG;AACtC,mBAAQ,KAA4B,WAAW,KAAK,KAAK;AAAA,UAC3D,CAAC,EAAE,MAAM,MAAM,EAAE;AAAA,QACnB;AACA,YAAI,CAAC,MAAO,SAAQ,QAAQ,MAAM;AAGlC,cAAM,aAA6B,MAAM,OAAO;AAAA,UAAS,CAAC,OACxD,MAAM,KAAK,GAAG,OAAO,EAAE,IAAI,QAAM,EAAE,OAAO,EAAE,OAAO,OAAO,EAAE,KAAK,KAAK,EAAE,EAAE;AAAA,QAC5E,EAAE,MAAM,MAAM,CAAC,CAAC;AAEhB,cAAM,UAAU,WAAW;AAAA,UACzB,OAAK,EAAE,2BAA2B,KAAK,EAAE,KAAK,KAAK,EAAE,UAAU;AAAA,QACjE;AAEA,YAAI,QAAQ,WAAW,EAAG;AAE1B,gBAAQ,KAAK,EAAE,MAAM,UAAU,OAAO,aAAa,QAAQ,CAAC;AAC5D,YAAI,QAAQ,UAAU,GAAI;AAAA,MAC5B,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AAGA,MAAI,QAAQ,SAAS,IAAI;AACvB,QAAI;AACF,YAAM,gBAAgB,MAAM,KACzB;AAAA,QACC;AAAA,MAEF,EACC,IAAI;AAEP,iBAAW,UAAU,cAAc,MAAM,GAAG,KAAK,QAAQ,MAAM,GAAG;AAChE,YAAI;AACF,gBAAM,KAAK,MAAM,OAAO,aAAa,IAAI,EAAE,MAAM,MAAM,IAAI;AAC3D,gBAAM,cAAc,MAAM,OAAO,aAAa,aAAa,EAAE,MAAM,MAAM,IAAI;AAC7E,gBAAM,cAAc,KAAK,IAAI,EAAE,KAAK,cAAc,sBAAsB,WAAW,OAAO;AAE1F,cAAI,QAAQ;AACZ,cAAI,IAAI;AACN,oBAAQ,MAAM,KAAK,QAAQ,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,MAAM,EAAE;AAAA,UAC7E;AACA,cAAI,CAAC,MAAO,SAAQ,eAAe;AAEnC,kBAAQ,KAAK,EAAE,MAAM,UAAU,OAAO,aAAa,SAAS,CAAC,EAAE,CAAC;AAChE,cAAI,QAAQ,UAAU,GAAI;AAAA,QAC5B,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,SAAO;AACT;AAMA,eAAe,sBACb,MACA,QACA,QACsB;AACtB,QAAM,QAAqB;AAAA,IACzB,aAAa,OAAO;AAAA,IACpB,aAAa,OAAO;AAAA,IACpB,aAAa;AAAA,IACb,gBAAgB,CAAC;AAAA,EACnB;AAEA,MAAI;AACF,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,KAAK,QAAQ,OAAO,WAAW,EAAE,aAAa,OAAO,KAAK;AAAA,IAClE,WAAW,OAAO,SAAS,UAAU;AACnC,YAAM,KAAK,QAAQ,OAAO,WAAW,EAAE,KAAK,OAAO,KAAK;AACxD,YAAM,KAAK,QAAQ,OAAO,WAAW,EAAE,MAAM,OAAO;AAAA,IACtD;AAEA,UAAM,cAAc,MAAM,IAAI;AAE9B,UAAM,cAAc,MAAM,gBAAgB,IAAI;AAC9C,UAAM,iBAAiB,MAAM,gBAAgB,IAAI;AAAA,EACnD,QAAQ;AAAA,EAER;AAGA,MAAI;AACF,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,cAAc,MAAM,KAAK,QAAQ,OAAO,WAAW,EAAE;AAAA,QACzD,CAAC,OAA0B,GAAG,QAAQ,CAAC,GAAG,SAAS;AAAA,MACrD;AACA,YAAM,KAAK,QAAQ,OAAO,WAAW,EAAE,aAAa,WAAW;AAAA,IACjE,WAAW,OAAO,SAAS,UAAU;AACnC,YAAM,KAAK,QAAQ,OAAO,WAAW,EAAE,KAAK,EAAE;AAC9C,YAAM,KAAK,QAAQ,OAAO,WAAW,EAAE,MAAM,OAAO;AAAA,IACtD;AACA,UAAM,cAAc,MAAM,GAAG;AAAA,EAC/B,QAAQ;AAAA,EAER;AAEA,SAAO;AACT;AAMA,eAAe,kBAAkB,MAAY,SAA8C;AACzF,QAAM,SAAwB,CAAC;AAC/B,MAAI,cAAc;AAElB,aAAW,UAAU,SAAS;AAC5B,QAAI,eAAe,GAAI;AAEvB,UAAM,EAAE,QAAQ,IAAI;AACpB,QAAI,QAAQ,WAAW,EAAG;AAG1B,UAAM,UAAU,oBAAI,IAAY;AAAA,MAC9B;AAAA,MACA,KAAK,OAAO,QAAQ,SAAS,KAAK,CAAC;AAAA,MACnC,QAAQ,SAAS;AAAA,IACnB,CAAC;AACD,UAAM,SAAS,MAAM,KAAK,OAAO,EAAE,MAAM,GAAG,CAAC;AAE7C,eAAW,OAAO,QAAQ;AACxB,UAAI,eAAe,GAAI;AACvB,YAAM,SAAS,QAAQ,GAAG;AAC1B,UAAI;AACF,cAAM,IAAI,MAAM,sBAAsB,MAAM,QAAQ,MAAM;AAC1D,eAAO,KAAK,CAAC;AAAA,MACf,QAAQ;AAAA,MAER;AACA;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAMA,eAAe,qBACb,MACgE;AAChE,MAAI;AAEF,UAAM,gBAAgB,MAAM,KACzB,QAAQ,wEAAwE,EAChF,MAAM,EACN,MAAM,MAAM,CAAC;AAChB,QAAI,gBAAgB,EAAG,QAAO;AAG9B,UAAM,WAAW,MAAM,KACpB,QAAQ,qFAAqF,EAC7F,MAAM,EACN,MAAM,MAAM,CAAC;AAChB,QAAI,WAAW,EAAG,QAAO;AAGzB,UAAM,aAAa,MAAM,KAAK,SAAS,MAAM;AAC3C,aAAO,SAAS,gBAAgB,eAAe,OAAO,cAAc;AAAA,IACtE,CAAC,EAAE,MAAM,MAAM,KAAK;AACpB,QAAI,WAAY,QAAO;AAAA,EACzB,QAAQ;AAAA,EAER;AAEA,SAAO;AACT;AAMA,eAAe,wBAAwB,MAAwC;AAC7E,QAAM,QAAyD,CAAC;AAChE,QAAM,YAAsB,CAAC;AAG7B,MAAI;AACF,UAAM,aAAa,KAAK,QAAQ,iFAAiF;AACjH,UAAM,YAAY,MAAM,WAAW,MAAM;AACzC,QAAI,YAAY,GAAG;AACjB,YAAM,WAAW,MAAM,WAAW,MAAM,EAAE;AAAA,QAAS,CAAC,OAClD,MAAM,KAAK,GAAG,OAAO,EAAE,IAAI,OAAK,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,OAAK,CAAC,MAAM,CAAC,CAAC;AAAA,MAC9E;AACA,gBAAU,KAAK,GAAG,QAAQ;AAAA,IAC5B;AAAA,EACF,QAAQ;AAAA,EAER;AAGA,MAAI;AACJ,MAAI;AACF,iBAAa,MAAM,KAAK,SAAS,MAAM;AACrC,YAAM,UAAU,MAAM;AAAA,QACpB,SAAS,iBAAiB,4CAA4C;AAAA,MACxE;AACA,YAAM,OAAO,QACV,IAAI,OAAK,SAAU,EAAkB,UAAU,KAAK,GAAG,EAAE,CAAC,EAC1D,OAAO,OAAK,CAAC,MAAM,CAAC,CAAC;AACxB,aAAO,KAAK,SAAS,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI;AAAA,IAC/C,CAAC;AAAA,EACH,QAAQ;AAAA,EAER;AAGA,QAAM,KAAK,EAAE,SAAS,GAAG,WAAW,MAAM,gBAAgB,IAAI,EAAE,CAAC;AAGjE,WAAS,UAAU,GAAG,WAAW,IAAI,WAAW;AAC9C,QAAI;AACF,YAAM,UAAU,KAAK;AAAA,QACnB;AAAA,MACF,EAAE,MAAM;AAER,YAAM,aAAa,MAAM,QAAQ,WAAW,EAAE,MAAM,MAAM,IAAI;AAC9D,UAAI,WAAY;AAEhB,YAAM,QAAQ,MAAM;AACpB,YAAM,cAAc,MAAM,GAAI;AAE9B,YAAM,YAAY,MAAM,gBAAgB,IAAI;AAC5C,YAAM,KAAK,EAAE,SAAS,UAAU,CAAC;AAAA,IACnC,QAAQ;AACN;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,WAAW,UAAU,SAAS,IAAI,YAAY;AAAA,IAC9C;AAAA,EACF;AACF;AAMA,eAAe,sBAAsB,MAAwC;AAC3E,QAAM,QAAyD,CAAC;AAEhE,MAAI,eAAe;AACnB,MAAI,YAAY;AAEhB,SAAO,YAAY,IAAI;AACrB,QAAI;AACF,YAAM,WAAW,MAAM,KAAK;AAAA,QAAS,MACnC,SAAS,iBAAiB,UAAU,EAAE;AAAA,MACxC,EAAE,MAAM,MAAM,CAAC;AAEf,UAAI,aAAa,aAAc;AAC/B,qBAAe;AAGf,YAAM,KAAK,EAAE,SAAS,YAAY,GAAG,WAAW,MAAM,gBAAgB,IAAI,EAAE,CAAC;AAE7E,YAAM,KAAK,SAAS,MAAM,OAAO,SAAS,GAAG,SAAS,KAAK,YAAY,CAAC;AACxE,YAAM,IAAI,QAAc,OAAK,WAAW,GAAG,GAAG,CAAC;AAAA,IACjD,QAAQ;AACN;AAAA,IACF;AACA;AAAA,EACF;AAEA,QAAM,aAAa,MAAM,KAAK;AAAA,IAAS,MACrC,SAAS,iBAAiB,UAAU,EAAE;AAAA,EACxC,EAAE,MAAM,MAAM,MAAS;AAEvB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,YAAY,OAAO,eAAe,WAAW,aAAa;AAAA,IAC1D;AAAA,EACF;AACF;AAMA,eAAe,mBACb,MACuE;AACvE,QAAM,UAAwE,CAAC;AAE/E,MAAI;AACF,UAAM,WAAW,MAAM,KAAK,QAAQ,aAAa,EAAE,IAAI;AACvD,eAAW,MAAM,SAAS,MAAM,GAAG,CAAC,GAAG;AACrC,UAAI;AACF,cAAM,SAAS,MAAM,GAAG,UAAU,EAAE,MAAM,MAAM,EAAE;AAClD,cAAM,GAAG,MAAM;AACf,cAAM,cAAc,MAAM,GAAG;AAC7B,cAAM,qBAAqB,MAAM,GAAG,aAAa,WAAW,EAAE,MAAM,MAAM,IAAI;AAC9E,gBAAQ,KAAK,EAAE,QAAQ,OAAO,KAAK,GAAG,mBAAmB,CAAC;AAG1D,cAAM,GAAG,MAAM,EAAE,MAAM,MAAM,MAAS;AACtC,cAAM,cAAc,MAAM,GAAG;AAAA,MAC/B,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AAEA,SAAO;AACT;AAMA,eAAsB,0BAA0B,MAA6C;AAE3F,MAAI,UAAuB,CAAC;AAC5B,MAAI;AACF,cAAU,MAAM,gBAAgB,IAAI;AAAA,EACtC,QAAQ;AAAA,EAER;AAGA,MAAI,eAA8B,CAAC;AACnC,MAAI;AACF,mBAAe,MAAM,kBAAkB,MAAM,OAAO;AAAA,EACtD,QAAQ;AAAA,EAER;AAGA,MAAI,aAAgC,EAAE,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC9D,MAAI;AACF,UAAM,OAAO,MAAM,qBAAqB,IAAI;AAE5C,QAAI,SAAS,YAAY;AACvB,mBAAa,MAAM,wBAAwB,IAAI;AAAA,IACjD,WAAW,SAAS,aAAa;AAE/B,mBAAa,MAAM,sBAAsB,IAAI;AAC7C,mBAAa,EAAE,GAAG,YAAY,MAAM,YAAY;AAAA,IAClD,WAAW,SAAS,mBAAmB;AACrC,mBAAa,MAAM,sBAAsB,IAAI;AAAA,IAC/C,OAAO;AAEL,YAAM,YAAY,MAAM,gBAAgB,IAAI;AAC5C,mBAAa,EAAE,MAAM,QAAQ,OAAO,UAAU,SAAS,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE;AAAA,IAC9F;AAAA,EACF,QAAQ;AAAA,EAER;AAGA,MAAI,cAA4E,CAAC;AACjF,MAAI;AACF,kBAAc,MAAM,mBAAmB,IAAI;AAAA,EAC7C,QAAQ;AAAA,EAER;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/frontier.js

@@ -2,11 +2,13 @@ function scoreUrl(url, keywords) {
   try {
     const u = new URL(url);
     const segments = u.pathname.split("/").filter(Boolean);
-    let score = -segments.length;
+    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? "") || 1;
+    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? "") || 10;
+    let score = -segments.length * depthPenaltyWeight;
     if (keywords.length > 0) {
       const lower = url.toLowerCase();
       for (const kw of keywords) {
-        if (kw && lower.includes(kw)) score += 10;
+        if (kw && lower.includes(kw)) score += keywordBoostWeight;
       }
     }
     return score;
@@ -41,7 +43,7 @@ class CrawlFrontier {
    */
   push(url, pushOpts) {
     if (this.urlFilter && !this.urlFilter.test(url)) return;
-    const sparseBoost = this.strategy === "BEST_FIRST" && pushOpts?.parentElements !== void 0 && pushOpts.parentElements < 5 ? 1 : 0;
+    const sparseBoost = this.strategy === "BEST_FIRST" && pushOpts?.parentElements !== void 0 && pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? "") || 5) ? 1 : 0;
     this.items.push({ url, score: this.strategy === "BEST_FIRST" ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
   }
   /** Dequeues the next URL per this frontier's strategy. undefined if empty. */

package/dist/frontier.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/frontier.ts"],
-  "sourcesContent": ["/**\n * Configurable crawl traversal strategy \u2014 matches crawl4AI's BFS/DFS/\n * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded\n * FIFO queue, i.e. always BFS).\n *\n * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it\n * drops into crawler.ts's existing queue call sites with a rename, not a\n * rewrite \u2014 BFS's `shift()` behavior is byte-for-byte the same as the plain\n * array it replaces, so BFS (the default) has zero behavior change.\n */\n\nexport type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';\n\ninterface FrontierEntry {\n  url: string;\n  score: number;\n}\n\n/**\n * Best-First scoring heuristic: shallower paths score higher (breadth-like\n * bias toward top-level sections before deep sub-pages), boosted further by\n * any user-supplied keyword appearing in the URL. Simple and transparent by\n * design \u2014 crawl4AI's own default scorer is a similarly simple keyword-\n * relevance heuristic, not a learned model.\n */\nfunction scoreUrl(url: string, keywords: string[]): number {\n  try {\n    const u = new URL(url);\n    const segments = u.pathname.split('/').filter(Boolean);\n    let score = -segments.length;\n    if (keywords.length > 0) {\n      const lower = url.toLowerCase();\n      for (const kw of keywords) {\n        if (kw && lower.includes(kw)) score += 10;\n      }\n    }\n    return score;\n  } catch {\n    return 0;\n  }\n}\n\nexport class CrawlFrontier {\n  private items: FrontierEntry[] = [];\n  private readonly strategy: TraversalStrategy;\n  private readonly keywords: string[];\n  private readonly urlFilter: RegExp | null;\n\n  constructor(strategy: TraversalStrategy, seedUrls: string[], keywords: string[] = [], urlFilter?: string) {\n    this.strategy = strategy;\n    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n    this.urlFilter = urlFilter\n      ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()\n      : null;\n    for (const url of seedUrls) this.push(url);\n  }\n\n  get length(): number {\n    return this.items.length;\n  }\n\n  /**\n   * P3b: sparse-parent boost \u2014 if the parent page had fewer than 5 interactive\n   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so\n   * the frontier revisits sparse branches before deep-linked content.\n   */\n  push(url: string, pushOpts?: { parentElements?: number }): void {\n    if (this.urlFilter && !this.urlFilter.test(url)) return;\n    const sparseBoost =\n      this.strategy === 'BEST_FIRST' &&\n      pushOpts?.parentElements !== undefined &&\n      pushOpts.parentElements < 5\n        ? 1\n        : 0;\n    this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });\n  }\n\n  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */\n  shift(): string | undefined {\n    if (this.items.length === 0) return undefined;\n    switch (this.strategy) {\n      case 'DFS':\n        return this.items.pop()!.url;\n      case 'BEST_FIRST': {\n        let bestIdx = 0;\n        for (let i = 1; i < this.items.length; i++) {\n          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;\n        }\n        return this.items.splice(bestIdx, 1)[0].url;\n      }\n      case 'BFS':\n      default:\n        return this.items.shift()!.url;\n    }\n  }\n\n  filter(predicate: (url: string) => boolean): string[] {\n    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);\n  }\n\n  /** Remove queued URLs not matching the regex pattern. Returns count removed. */\n  filterByPattern(pattern: string): number {\n    try {\n      const re = new RegExp(pattern, 'i');\n      const before = this.items.length;\n      this.items = this.items.filter((e) => re.test(e.url));\n      return before - this.items.length;\n    } catch {\n      return 0;\n    }\n  }\n}\n"],
-  "mappings": "AAyBA,SAAS,SAAS,KAAa,UAA4B;AACzD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,WAAW,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AACrD,QAAI,QAAQ,CAAC,SAAS;AACtB,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,QAAQ,IAAI,YAAY;AAC9B,iBAAW,MAAM,UAAU;AACzB,YAAI,MAAM,MAAM,SAAS,EAAE,EAAG,UAAS;AAAA,MACzC;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAc;AAAA,EACjB,QAAyB,CAAC;AAAA,EACjB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAA6B,UAAoB,WAAqB,CAAC,GAAG,WAAoB;AACxG,SAAK,WAAW;AAChB,SAAK,WAAW,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACnE,SAAK,YAAY,aACZ,MAAM;AAAE,UAAI;AAAE,eAAO,IAAI,OAAO,WAAW,GAAG;AAAA,MAAG,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IAAE,GAAG,IAC9E;AACJ,eAAW,OAAO,SAAU,MAAK,KAAK,GAAG;AAAA,EAC3C;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KAAK,KAAa,UAA8C;AAC9D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,KAAK,GAAG,EAAG;AACjD,UAAM,cACJ,KAAK,aAAa,gBAClB,UAAU,mBAAmB,UAC7B,SAAS,iBAAiB,IACtB,IACA;AACN,SAAK,MAAM,KAAK,EAAE,KAAK,OAAO,KAAK,aAAa,eAAe,SAAS,KAAK,KAAK,QAAQ,IAAI,cAAc,EAAE,CAAC;AAAA,EACjH;AAAA;AAAA,EAGA,QAA4B;AAC1B,QAAI,KAAK,MAAM,WAAW,EAAG,QAAO;AACpC,YAAQ,KAAK,UAAU;AAAA,MACrB,KAAK;AACH,eAAO,KAAK,MAAM,IAAI,EAAG;AAAA,MAC3B,KAAK,cAAc;AACjB,YAAI,UAAU;AACd,iBAAS,IAAI,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AAC1C,cAAI,KAAK,MAAM,CAAC,EAAE,QAAQ,KAAK,MAAM,OAAO,EAAE,MAAO,WAAU;AAAA,QACjE;AACA,eAAO,KAAK,MAAM,OAAO,SAAS,CAAC,EAAE,CAAC,EAAE;AAAA,MAC1C;AAAA,MACA,KAAK;AAAA,MACL;AACE,eAAO,KAAK,MAAM,MAAM,EAAG;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,OAAO,WAA+C;AACpD,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,UAAU,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG;AAAA,EACpE;AAAA;AAAA,EAGA,gBAAgB,SAAyB;AACvC,QAAI;AACF,YAAM,KAAK,IAAI,OAAO,SAAS,GAAG;AAClC,YAAM,SAAS,KAAK,MAAM;AAC1B,WAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC;AACpD,aAAO,SAAS,KAAK,MAAM;AAAA,IAC7B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
+  "sourcesContent": ["/**\n * Configurable crawl traversal strategy \u2014 matches crawl4AI's BFS/DFS/\n * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded\n * FIFO queue, i.e. always BFS).\n *\n * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it\n * drops into crawler.ts's existing queue call sites with a rename, not a\n * rewrite \u2014 BFS's `shift()` behavior is byte-for-byte the same as the plain\n * array it replaces, so BFS (the default) has zero behavior change.\n */\n\nexport type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';\n\ninterface FrontierEntry {\n  url: string;\n  score: number;\n}\n\n/**\n * Best-First scoring heuristic: shallower paths score higher (breadth-like\n * bias toward top-level sections before deep sub-pages), boosted further by\n * any user-supplied keyword appearing in the URL. Simple and transparent by\n * design \u2014 crawl4AI's own default scorer is a similarly simple keyword-\n * relevance heuristic, not a learned model.\n */\nfunction scoreUrl(url: string, keywords: string[]): number {\n  try {\n    const u = new URL(url);\n    const segments = u.pathname.split('/').filter(Boolean);\n    const depthPenaltyWeight = parseFloat(process.env.CRAWLER_FRONTIER_DEPTH_PENALTY ?? '') || 1;\n    const keywordBoostWeight = parseFloat(process.env.CRAWLER_FRONTIER_KEYWORD_BOOST ?? '') || 10;\n    let score = -segments.length * depthPenaltyWeight;\n    if (keywords.length > 0) {\n      const lower = url.toLowerCase();\n      for (const kw of keywords) {\n        if (kw && lower.includes(kw)) score += keywordBoostWeight;\n      }\n    }\n    return score;\n  } catch {\n    return 0;\n  }\n}\n\nexport class CrawlFrontier {\n  private items: FrontierEntry[] = [];\n  private readonly strategy: TraversalStrategy;\n  private readonly keywords: string[];\n  private readonly urlFilter: RegExp | null;\n\n  constructor(strategy: TraversalStrategy, seedUrls: string[], keywords: string[] = [], urlFilter?: string) {\n    this.strategy = strategy;\n    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n    this.urlFilter = urlFilter\n      ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()\n      : null;\n    for (const url of seedUrls) this.push(url);\n  }\n\n  get length(): number {\n    return this.items.length;\n  }\n\n  /**\n   * P3b: sparse-parent boost \u2014 if the parent page had fewer than 5 interactive\n   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so\n   * the frontier revisits sparse branches before deep-linked content.\n   */\n  push(url: string, pushOpts?: { parentElements?: number }): void {\n    if (this.urlFilter && !this.urlFilter.test(url)) return;\n    const sparseBoost =\n      this.strategy === 'BEST_FIRST' &&\n      pushOpts?.parentElements !== undefined &&\n      pushOpts.parentElements < (parseInt(process.env.CRAWLER_FRONTIER_SPARSE_PARENT_THRESHOLD ?? '') || 5)\n        ? 1\n        : 0;\n    this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });\n  }\n\n  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */\n  shift(): string | undefined {\n    if (this.items.length === 0) return undefined;\n    switch (this.strategy) {\n      case 'DFS':\n        return this.items.pop()!.url;\n      case 'BEST_FIRST': {\n        let bestIdx = 0;\n        for (let i = 1; i < this.items.length; i++) {\n          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;\n        }\n        return this.items.splice(bestIdx, 1)[0].url;\n      }\n      case 'BFS':\n      default:\n        return this.items.shift()!.url;\n    }\n  }\n\n  filter(predicate: (url: string) => boolean): string[] {\n    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);\n  }\n\n  /** Remove queued URLs not matching the regex pattern. Returns count removed. */\n  filterByPattern(pattern: string): number {\n    try {\n      const re = new RegExp(pattern, 'i');\n      const before = this.items.length;\n      this.items = this.items.filter((e) => re.test(e.url));\n      return before - this.items.length;\n    } catch {\n      return 0;\n    }\n  }\n}\n"],
+  "mappings": "AAyBA,SAAS,SAAS,KAAa,UAA4B;AACzD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,WAAW,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AACrD,UAAM,qBAAqB,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AAC3F,UAAM,qBAAqB,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AAC3F,QAAI,QAAQ,CAAC,SAAS,SAAS;AAC/B,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,QAAQ,IAAI,YAAY;AAC9B,iBAAW,MAAM,UAAU;AACzB,YAAI,MAAM,MAAM,SAAS,EAAE,EAAG,UAAS;AAAA,MACzC;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAc;AAAA,EACjB,QAAyB,CAAC;AAAA,EACjB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAA6B,UAAoB,WAAqB,CAAC,GAAG,WAAoB;AACxG,SAAK,WAAW;AAChB,SAAK,WAAW,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACnE,SAAK,YAAY,aACZ,MAAM;AAAE,UAAI;AAAE,eAAO,IAAI,OAAO,WAAW,GAAG;AAAA,MAAG,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IAAE,GAAG,IAC9E;AACJ,eAAW,OAAO,SAAU,MAAK,KAAK,GAAG;AAAA,EAC3C;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KAAK,KAAa,UAA8C;AAC9D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,KAAK,GAAG,EAAG;AACjD,UAAM,cACJ,KAAK,aAAa,gBAClB,UAAU,mBAAmB,UAC7B,SAAS,kBAAkB,SAAS,QAAQ,IAAI,4CAA4C,EAAE,KAAK,KAC/F,IACA;AACN,SAAK,MAAM,KAAK,EAAE,KAAK,OAAO,KAAK,aAAa,eAAe,SAAS,KAAK,KAAK,QAAQ,IAAI,cAAc,EAAE,CAAC;AAAA,EACjH;AAAA;AAAA,EAGA,QAA4B;AAC1B,QAAI,KAAK,MAAM,WAAW,EAAG,QAAO;AACpC,YAAQ,KAAK,UAAU;AAAA,MACrB,KAAK;AACH,eAAO,KAAK,MAAM,IAAI,EAAG;AAAA,MAC3B,KAAK,cAAc;AACjB,YAAI,UAAU;AACd,iBAAS,IAAI,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AAC1C,cAAI,KAAK,MAAM,CAAC,EAAE,QAAQ,KAAK,MAAM,OAAO,EAAE,MAAO,WAAU;AAAA,QACjE;AACA,eAAO,KAAK,MAAM,OAAO,SAAS,CAAC,EAAE,CAAC,EAAE;AAAA,MAC1C;AAAA,MACA,KAAK;AAAA,MACL;AACE,eAAO,KAAK,MAAM,MAAM,EAAG;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,OAAO,WAA+C;AACpD,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,UAAU,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG;AAAA,EACpE;AAAA;AAAA,EAGA,gBAAgB,SAAyB;AACvC,QAAI;AACF,YAAM,KAAK,IAAI,OAAO,SAAS,GAAG;AAClC,YAAM,SAAS,KAAK,MAAM;AAC1B,WAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC;AACpD,aAAO,SAAS,KAAK,MAAM;AAAA,IAC7B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/gap-pass.js

@@ -1,7 +1,23 @@
 import { logger as rootLogger } from "./logger.js";
 const logger = rootLogger.child({ module: "gap-pass" });
+function resolveTextLLM(llmCfg) {
+  if (llmCfg?.provider === "openai" && llmCfg.apiKey) {
+    return { provider: "openai", apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };
+  }
+  if (llmCfg?.provider === "anthropic" && llmCfg.apiKey) {
+    return { provider: "anthropic", apiKey: llmCfg.apiKey, model: llmCfg.model };
+  }
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: "openai", apiKey: process.env.OPENAI_API_KEY, model: "gpt-4o-mini" };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { provider: "anthropic", apiKey: process.env.ANTHROPIC_API_KEY, model: "claude-haiku-4-5-20251001" };
+  }
+  return null;
+}
 async function runGapAnalysis(screens, appUrl, llmCfg) {
-  if (!llmCfg?.apiKey || screens.length < 2) return [];
+  const effective = resolveTextLLM(llmCfg);
+  if (!effective || screens.length < 2) return [];
   const origin = (() => {
     try {
       return new URL(appUrl).origin;
@@ -17,7 +33,8 @@ async function runGapAnalysis(screens, appUrl, llmCfg) {
       return s.url;
     }
   }).join("\n");
-  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
+  const systemPrompt = "You are a web application coverage analyst. Respond with valid JSON only.";
+  const userPrompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
 
 Discovered screens (${screens.length} total):
 ${screenList}
@@ -29,23 +46,39 @@ Based on these screens and common patterns for this type of web application, ide
 Respond with ONLY valid JSON, no markdown:
 {"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`;
   try {
-    const Anthropic = (await import("@anthropic-ai/sdk")).default;
-    const client = new Anthropic({ apiKey: llmCfg.apiKey });
-    const response = await client.messages.create({
-      model: llmCfg.model ?? "claude-haiku-4-5-20251001",
-      max_tokens: 1024,
-      temperature: 0,
-      // @ts-ignore
-      system: [{ type: "text", text: "You are a web application coverage analyst. Respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
-      messages: [{ role: "user", content: prompt }]
-    });
-    const text = response.content[0]?.text ?? "";
+    let text = "";
+    if (effective.provider === "openai") {
+      const { OpenAI } = await import("openai");
+      const client = new OpenAI({ apiKey: effective.apiKey, ...effective.baseUrl ? { baseURL: effective.baseUrl } : {} });
+      const response = await client.chat.completions.create({
+        model: effective.model ?? "gpt-4o-mini",
+        max_tokens: 1024,
+        temperature: 0,
+        messages: [
+          { role: "system", content: systemPrompt },
+          { role: "user", content: userPrompt }
+        ]
+      });
+      text = response.choices[0]?.message?.content ?? "";
+    } else {
+      const Anthropic = (await import("@anthropic-ai/sdk")).default;
+      const client = new Anthropic({ apiKey: effective.apiKey });
+      const response = await client.messages.create({
+        model: effective.model ?? "claude-haiku-4-5-20251001",
+        max_tokens: 1024,
+        temperature: 0,
+        // @ts-ignore
+        system: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }],
+        messages: [{ role: "user", content: userPrompt }]
+      });
+      text = response.content[0]?.text ?? "";
+    }
     const jsonMatch = text.match(/\{[\s\S]*\}/);
     if (!jsonMatch) return [];
     const parsed = JSON.parse(jsonMatch[0]);
     const rawGaps = parsed.gaps ?? [];
     if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];
-    logger.info({ gaps: rawGaps.length }, "[P6] Gap analysis: LLM returned candidates");
+    logger.info({ gaps: rawGaps.length, provider: effective.provider }, "[P6] Gap analysis: LLM returned candidates");
     const results = [];
     for (const gap of rawGaps.slice(0, 15)) {
       let confirmedUrl = null;

package/dist/gap-pass.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/gap-pass.ts"],
-  "sourcesContent": ["/**\n * P6: Gap Pass \u2014 LLM-guided second crawl pass\n *\n * After the primary BFS crawl completes, this module:\n * 1. Sends the discovered screen list to an LLM for gap analysis\n * 2. HEAD-validates each suggested URL\n * 3. Returns confirmed gap URLs for targeted re-crawl\n *\n * The caller (crawler.ts) is responsible for actually visiting the confirmed\n * URLs with the existing browser \u2014 this module only identifies them.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'gap-pass' });\n\nexport interface GapCandidate {\n  category: string;\n  urlPatterns: string[];\n  confidence: number;\n  rationale: string;\n  confirmedUrl: string | null;\n  headStatus: number | null;\n}\n\ninterface DiscoveredScreen {\n  url: string;\n  name?: string;\n  elements?: number;\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n}\n\n/**\n * Run LLM gap analysis + HEAD validation.\n * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.\n */\nexport async function runGapAnalysis(\n  screens: DiscoveredScreen[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<GapCandidate[]> {\n  if (!llmCfg?.apiKey || screens.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Cap context \u2014 only send url + name, not full element data\n  const screenList = screens\n    .slice(0, 80)\n    .map(s => {\n      try { return new URL(s.url).pathname + (s.name ? ` (${s.name})` : ''); } catch { return s.url; }\n    })\n    .join('\\n');\n\n  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.\n\nDiscovered screens (${screens.length} total):\n${screenList}\n\nApp URL: ${appUrl}\n\nBased on these screens and common patterns for this type of web application, identify screen categories that are likely present but missing. Focus on pages that would exist in a typical SaaS or web app but are not represented above.\n\nRespond with ONLY valid JSON, no markdown:\n{\"gaps\":[{\"category\":\"User profile edit\",\"urlPatterns\":[\"/profile/edit\",\"/account/profile\"],\"confidence\":0.8,\"rationale\":\"Profile view found but no edit page\"},{\"category\":\"Billing history\",\"urlPatterns\":[\"/settings/billing\",\"/account/billing\"],\"confidence\":0.7,\"rationale\":\"Settings section found but no billing page\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      temperature: 0,\n      // @ts-ignore\n      system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawGaps: Array<{ category: string; urlPatterns: string[]; confidence: number; rationale: string }> =\n      parsed.gaps ?? [];\n\n    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];\n    logger.info({ gaps: rawGaps.length }, '[P6] Gap analysis: LLM returned candidates');\n\n    // HEAD-validate each URL pattern, take the first 200\n    const results: GapCandidate[] = [];\n    for (const gap of rawGaps.slice(0, 15)) {\n      let confirmedUrl: string | null = null;\n      let headStatus: number | null = null;\n\n      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {\n        const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`;\n        try {\n          const controller = new AbortController();\n          const timeout = setTimeout(() => controller.abort(), 6_000);\n          const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n          clearTimeout(timeout);\n          headStatus = res?.status ?? null;\n          if (res?.ok) { confirmedUrl = fullUrl; break; }\n        } catch { /* non-fatal */ }\n      }\n\n      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });\n      logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked');\n    }\n\n    const confirmed = results.filter(r => r.confirmedUrl).length;\n    logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[P6] Gap analysis failed \u2014 non-fatal');\n    return [];\n  }\n}\n"],
-  "mappings": "AAYA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,WAAW,CAAC;AA2BtD,eAAsB,eACpB,SACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,QAAQ,SAAS,EAAG,QAAO,CAAC;AAEnD,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,aAAa,QAChB,MAAM,GAAG,EAAE,EACX,IAAI,OAAK;AACR,QAAI;AAAE,aAAO,IAAI,IAAI,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,KAAK,EAAE,IAAI,MAAM;AAAA,IAAK,QAAQ;AAAE,aAAO,EAAE;AAAA,IAAK;AAAA,EACjG,CAAC,EACA,KAAK,IAAI;AAEZ,QAAM,SAAS;AAAA;AAAA,sBAEK,QAAQ,MAAM;AAAA,EAClC,UAAU;AAAA;AAAA,WAED,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAOf,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA,MACZ,aAAa;AAAA;AAAA,MAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,6EAA6E,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAClJ,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,UACJ,OAAO,QAAQ,CAAC;AAElB,QAAI,CAAC,MAAM,QAAQ,OAAO,KAAK,QAAQ,WAAW,EAAG,QAAO,CAAC;AAC7D,WAAO,KAAK,EAAE,MAAM,QAAQ,OAAO,GAAG,4CAA4C;AAGlF,UAAM,UAA0B,CAAC;AACjC,eAAW,OAAO,QAAQ,MAAM,GAAG,EAAE,GAAG;AACtC,UAAI,eAA8B;AAClC,UAAI,aAA4B;AAEhC,iBAAW,YAAY,IAAI,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG;AACzD,cAAM,UAAU,QAAQ,WAAW,MAAM,IAAI,UAAU,GAAG,MAAM,GAAG,QAAQ,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,OAAO;AAC/G,YAAI;AACF,gBAAM,aAAa,IAAI,gBAAgB;AACvC,gBAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,gBAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,uBAAa,OAAO;AACpB,uBAAa,KAAK,UAAU;AAC5B,cAAI,KAAK,IAAI;AAAE,2BAAe;AAAS;AAAA,UAAO;AAAA,QAChD,QAAQ;AAAA,QAAkB;AAAA,MAC5B;AAEA,cAAQ,KAAK,EAAE,UAAU,IAAI,UAAU,aAAa,IAAI,aAAa,YAAY,IAAI,YAAY,WAAW,IAAI,WAAW,cAAc,WAAW,CAAC;AACrJ,aAAO,KAAK,EAAE,UAAU,IAAI,UAAU,cAAc,WAAW,GAAG,4BAA4B;AAAA,IAChG;AAEA,UAAM,YAAY,QAAQ,OAAO,OAAK,EAAE,YAAY,EAAE;AACtD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,UAAU,GAAG,iCAAiC;AACnF,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,2CAAsC;AACxE,WAAO,CAAC;AAAA,EACV;AACF;",
+  "sourcesContent": ["/**\n * P6: Gap Pass \u2014 LLM-guided second crawl pass\n *\n * After the primary BFS crawl completes, this module:\n * 1. Sends the discovered screen list to an LLM for gap analysis\n * 2. HEAD-validates each suggested URL\n * 3. Returns confirmed gap URLs for targeted re-crawl\n *\n * The caller (crawler.ts) is responsible for actually visiting the confirmed\n * URLs with the existing browser \u2014 this module only identifies them.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'gap-pass' });\n\nexport interface GapCandidate {\n  category: string;\n  urlPatterns: string[];\n  confidence: number;\n  rationale: string;\n  confirmedUrl: string | null;\n  headStatus: number | null;\n}\n\ninterface DiscoveredScreen {\n  url: string;\n  name?: string;\n  elements?: number;\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n// Only openai and anthropic are supported here. NVIDIA, groq, ollama, etc. \u2192 skip, fall to env.\n/**\n * Resolve effective LLM. Priority: openai \u2192 anthropic \u2192 skip unsupported providers.\n * Falls back to env vars when platform provider is not supported.\n */\nfunction resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null {\n  if (llmCfg?.provider === 'openai' && llmCfg.apiKey) {\n    return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };\n  }\n  if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) {\n    return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model };\n  }\n  // Unsupported provider (nvidia, groq, etc.) or no key \u2014 env fallback, openai first.\n  if (process.env.OPENAI_API_KEY) {\n    return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' };\n  }\n  if (process.env.ANTHROPIC_API_KEY) {\n    return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' };\n  }\n  return null;\n}\n\n/**\n * Run LLM gap analysis + HEAD validation.\n * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.\n */\nexport async function runGapAnalysis(\n  screens: DiscoveredScreen[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<GapCandidate[]> {\n  const effective = resolveTextLLM(llmCfg);\n  if (!effective || screens.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Cap context \u2014 only send url + name, not full element data\n  const screenList = screens\n    .slice(0, 80)\n    .map(s => {\n      try { return new URL(s.url).pathname + (s.name ? ` (${s.name})` : ''); } catch { return s.url; }\n    })\n    .join('\\n');\n\n  const systemPrompt = 'You are a web application coverage analyst. Respond with valid JSON only.';\n  const userPrompt = `You are analyzing the screen map of a web application to find structural coverage gaps.\n\nDiscovered screens (${screens.length} total):\n${screenList}\n\nApp URL: ${appUrl}\n\nBased on these screens and common patterns for this type of web application, identify screen categories that are likely present but missing. Focus on pages that would exist in a typical SaaS or web app but are not represented above.\n\nRespond with ONLY valid JSON, no markdown:\n{\"gaps\":[{\"category\":\"User profile edit\",\"urlPatterns\":[\"/profile/edit\",\"/account/profile\"],\"confidence\":0.8,\"rationale\":\"Profile view found but no edit page\"},{\"category\":\"Billing history\",\"urlPatterns\":[\"/settings/billing\",\"/account/billing\"],\"confidence\":0.7,\"rationale\":\"Settings section found but no billing page\"}]}`;\n\n  try {\n    let text = '';\n\n    if (effective.provider === 'openai') {\n      const { OpenAI } = await import('openai');\n      const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) });\n      const response = await client.chat.completions.create({\n        model: effective.model ?? 'gpt-4o-mini',\n        max_tokens: 1024,\n        temperature: 0,\n        messages: [\n          { role: 'system', content: systemPrompt },\n          { role: 'user', content: userPrompt },\n        ],\n      });\n      text = response.choices[0]?.message?.content ?? '';\n    } else {\n      const Anthropic = (await import('@anthropic-ai/sdk')).default;\n      const client = new Anthropic({ apiKey: effective.apiKey });\n      const response = await client.messages.create({\n        model: effective.model ?? 'claude-haiku-4-5-20251001',\n        max_tokens: 1024,\n        temperature: 0,\n        // @ts-ignore\n        system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],\n        messages: [{ role: 'user', content: userPrompt }],\n      });\n      text = (response.content[0] as any)?.text ?? '';\n    }\n\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawGaps: Array<{ category: string; urlPatterns: string[]; confidence: number; rationale: string }> =\n      parsed.gaps ?? [];\n\n    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];\n    logger.info({ gaps: rawGaps.length, provider: effective.provider }, '[P6] Gap analysis: LLM returned candidates');\n\n    // HEAD-validate each URL pattern\n    const results: GapCandidate[] = [];\n    for (const gap of rawGaps.slice(0, 15)) {\n      let confirmedUrl: string | null = null;\n      let headStatus: number | null = null;\n\n      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {\n        const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`;\n        try {\n          const controller = new AbortController();\n          const timeout = setTimeout(() => controller.abort(), 6_000);\n          const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n          clearTimeout(timeout);\n          headStatus = res?.status ?? null;\n          if (res?.ok) { confirmedUrl = fullUrl; break; }\n        } catch { /* non-fatal */ }\n      }\n\n      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });\n      logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked');\n    }\n\n    const confirmed = results.filter(r => r.confirmedUrl).length;\n    logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[P6] Gap analysis failed \u2014 non-fatal');\n    return [];\n  }\n}\n"],
+  "mappings": "AAYA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,WAAW,CAAC;AA6BtD,SAAS,eAAe,QAAiI;AACvJ,MAAI,QAAQ,aAAa,YAAY,OAAO,QAAQ;AAClD,WAAO,EAAE,UAAU,UAAU,QAAQ,OAAO,QAAQ,OAAO,OAAO,OAAO,SAAS,OAAO,QAAQ;AAAA,EACnG;AACA,MAAI,QAAQ,aAAa,eAAe,OAAO,QAAQ;AACrD,WAAO,EAAE,UAAU,aAAa,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM;AAAA,EAC7E;AAEA,MAAI,QAAQ,IAAI,gBAAgB;AAC9B,WAAO,EAAE,UAAU,UAAU,QAAQ,QAAQ,IAAI,gBAAgB,OAAO,cAAc;AAAA,EACxF;AACA,MAAI,QAAQ,IAAI,mBAAmB;AACjC,WAAO,EAAE,UAAU,aAAa,QAAQ,QAAQ,IAAI,mBAAmB,OAAO,4BAA4B;AAAA,EAC5G;AACA,SAAO;AACT;AAMA,eAAsB,eACpB,SACA,QACA,QACyB;AACzB,QAAM,YAAY,eAAe,MAAM;AACvC,MAAI,CAAC,aAAa,QAAQ,SAAS,EAAG,QAAO,CAAC;AAE9C,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,aAAa,QAChB,MAAM,GAAG,EAAE,EACX,IAAI,OAAK;AACR,QAAI;AAAE,aAAO,IAAI,IAAI,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,KAAK,EAAE,IAAI,MAAM;AAAA,IAAK,QAAQ;AAAE,aAAO,EAAE;AAAA,IAAK;AAAA,EACjG,CAAC,EACA,KAAK,IAAI;AAEZ,QAAM,eAAe;AACrB,QAAM,aAAa;AAAA;AAAA,sBAEC,QAAQ,MAAM;AAAA,EAClC,UAAU;AAAA;AAAA,WAED,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAOf,MAAI;AACF,QAAI,OAAO;AAEX,QAAI,UAAU,aAAa,UAAU;AACnC,YAAM,EAAE,OAAO,IAAI,MAAM,OAAO,QAAQ;AACxC,YAAM,SAAS,IAAI,OAAO,EAAE,QAAQ,UAAU,QAAQ,GAAI,UAAU,UAAU,EAAE,SAAS,UAAU,QAAQ,IAAI,CAAC,EAAG,CAAC;AACpH,YAAM,WAAW,MAAM,OAAO,KAAK,YAAY,OAAO;AAAA,QACpD,OAAO,UAAU,SAAS;AAAA,QAC1B,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,UAAU;AAAA,UACR,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,UACxC,EAAE,MAAM,QAAQ,SAAS,WAAW;AAAA,QACtC;AAAA,MACF,CAAC;AACD,aAAO,SAAS,QAAQ,CAAC,GAAG,SAAS,WAAW;AAAA,IAClD,OAAO;AACL,YAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,YAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,UAAU,OAAO,CAAC;AACzD,YAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,QAC5C,OAAO,UAAU,SAAS;AAAA,QAC1B,YAAY;AAAA,QACZ,aAAa;AAAA;AAAA,QAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,cAAc,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,QACnF,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,WAAW,CAAC;AAAA,MAClD,CAAC;AACD,aAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AAAA,IAC/C;AAEA,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,UACJ,OAAO,QAAQ,CAAC;AAElB,QAAI,CAAC,MAAM,QAAQ,OAAO,KAAK,QAAQ,WAAW,EAAG,QAAO,CAAC;AAC7D,WAAO,KAAK,EAAE,MAAM,QAAQ,QAAQ,UAAU,UAAU,SAAS,GAAG,4CAA4C;AAGhH,UAAM,UAA0B,CAAC;AACjC,eAAW,OAAO,QAAQ,MAAM,GAAG,EAAE,GAAG;AACtC,UAAI,eAA8B;AAClC,UAAI,aAA4B;AAEhC,iBAAW,YAAY,IAAI,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG;AACzD,cAAM,UAAU,QAAQ,WAAW,MAAM,IAAI,UAAU,GAAG,MAAM,GAAG,QAAQ,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,OAAO;AAC/G,YAAI;AACF,gBAAM,aAAa,IAAI,gBAAgB;AACvC,gBAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,gBAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,uBAAa,OAAO;AACpB,uBAAa,KAAK,UAAU;AAC5B,cAAI,KAAK,IAAI;AAAE,2BAAe;AAAS;AAAA,UAAO;AAAA,QAChD,QAAQ;AAAA,QAAkB;AAAA,MAC5B;AAEA,cAAQ,KAAK,EAAE,UAAU,IAAI,UAAU,aAAa,IAAI,aAAa,YAAY,IAAI,YAAY,WAAW,IAAI,WAAW,cAAc,WAAW,CAAC;AACrJ,aAAO,KAAK,EAAE,UAAU,IAAI,UAAU,cAAc,WAAW,GAAG,4BAA4B;AAAA,IAChG;AAEA,UAAM,YAAY,QAAQ,OAAO,OAAK,EAAE,YAAY,EAAE;AACtD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,UAAU,GAAG,iCAAiC;AACnF,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,2CAAsC;AACxE,WAAO,CAAC;AAAA,EACV;AACF;",
   "names": []
 }