@detiq/crawler 1.1.13 → 1.1.15

package/src/filter-pagination-crawler.ts

@@ -0,0 +1,500 @@
+/**
+ * Filter state crawling and pagination capture.
+ *
+ * Discovers filter controls (select, checkbox-group, radio-group, search),
+ * applies each meaningful option and records result counts and first-row
+ * values, then detects and walks pagination (numbered, load-more, infinite
+ * scroll). Also captures sort column behaviour.
+ *
+ * All per-filter / per-page errors are caught and skipped so one bad control
+ * never aborts the full crawl.
+ */
+
+import type { Page } from 'playwright';
+
+// ---------------------------------------------------------------------------
+// Public interfaces
+// ---------------------------------------------------------------------------
+
+export interface FilterOption {
+  value: string;
+  label: string;
+}
+
+export interface FilterDef {
+  kind: 'select' | 'checkbox-group' | 'radio-group' | 'search';
+  label: string;
+  locatorHint: string;
+  options: FilterOption[];
+}
+
+export interface FilterState {
+  filterLabel: string;
+  filterValue: string;
+  resultCount: number | null;
+  firstRowValues: string[];
+}
+
+export interface PaginationCapture {
+  kind: 'numbered' | 'load-more' | 'infinite-scroll' | 'none';
+  totalPages?: number;
+  totalItems?: number;
+  pageSizes?: number[];
+  pages: Array<{ pageNum: number; rowValues: string[] }>;
+}
+
+export interface FilterPaginationResult {
+  filters: FilterDef[];
+  filterStates: FilterState[];
+  pagination: PaginationCapture;
+  sortColumns: Array<{ header: string; ariaSortAfterClick: string | null }>;
+  searchCapture?: { noResultsText: string | null; sampleSuggestions: string[] };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/** Pause for up to `ms` ms or until network is idle, whichever comes first. */
+async function waitForSettle(page: Page, ms = 1500): Promise<void> {
+  await Promise.race([
+    page.waitForLoadState('networkidle').catch(() => undefined),
+    new Promise<void>(r => setTimeout(r, ms)),
+  ]);
+}
+
+/** Extract visible cell text from the first tbody row, if any. */
+async function captureFirstRow(page: Page): Promise<string[]> {
+  try {
+    return await page.evaluate(() => {
+      const row = document.querySelector('tbody tr');
+      if (!row) return [];
+      return Array.from(row.querySelectorAll('td')).map(td => td.innerText.trim());
+    });
+  } catch {
+    return [];
+  }
+}
+
+/** Read a result-count hint from visible page text using common patterns. */
+async function readResultCount(page: Page): Promise<number | null> {
+  try {
+    const text = await page.evaluate(() => document.body.innerText);
+    const match = text.match(/(\d[\d,]*)\s*(result|item|entr|record)/i);
+    if (!match) return null;
+    return parseInt(match[1].replace(/,/g, ''), 10);
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// discoverFilters
+// ---------------------------------------------------------------------------
+
+async function discoverFilters(page: Page): Promise<FilterDef[]> {
+  const filters: FilterDef[] = [];
+
+  // --- native <select> elements outside dialogs ---------------------------------
+  try {
+    const selectHandles = await page.locator('select:not([role="dialog"] select)').all();
+    for (const handle of selectHandles.slice(0, 10)) {
+      try {
+        const id = await handle.getAttribute('id').catch(() => null);
+        const name = await handle.getAttribute('name').catch(() => null);
+        const locatorHint = id ? `#${id}` : name ? `select[name="${name}"]` : 'select';
+
+        // Find associated label
+        let label = '';
+        if (id) {
+          label = await page.locator(`label[for="${id}"]`).innerText().catch(() => '');
+        }
+        if (!label) {
+          label = await handle.evaluate((el: HTMLSelectElement) => {
+            const lbl = el.closest('label') ?? el.previousElementSibling;
+            return (lbl as HTMLElement | null)?.innerText?.trim() ?? '';
+          }).catch(() => '');
+        }
+        if (!label) label = name ?? id ?? 'filter';
+
+        // Collect options, skipping default/placeholder entries
+        const rawOptions: FilterOption[] = await handle.evaluate((el: HTMLSelectElement) =>
+          Array.from(el.options).map(o => ({ value: o.value, label: o.text.trim() })),
+        ).catch(() => []);
+
+        const options = rawOptions.filter(
+          o => !(/^all$|^any$|^select|^--/i.test(o.label) || o.value === ''),
+        );
+
+        if (options.length === 0) continue;
+
+        filters.push({ kind: 'select', label, locatorHint, options });
+        if (filters.length >= 10) break;
+      } catch {
+        // skip this select
+      }
+    }
+  } catch {
+    // page.locator failed
+  }
+
+  // --- search inputs ------------------------------------------------------------
+  if (filters.length < 10) {
+    try {
+      const searchHandles = await page
+        .locator(
+          'input[type="search"]:not([role="dialog"] input), ' +
+          'input[placeholder*="search" i]:not([role="dialog"] input)',
+        )
+        .all();
+
+      for (const handle of searchHandles.slice(0, 10 - filters.length)) {
+        try {
+          const id = await handle.getAttribute('id').catch(() => null);
+          const placeholder = await handle.getAttribute('placeholder').catch(() => null);
+          const locatorHint = id ? `#${id}` : placeholder ? `input[placeholder="${placeholder}"]` : 'input[type="search"]';
+
+          let label = '';
+          if (id) {
+            label = await page.locator(`label[for="${id}"]`).innerText().catch(() => '');
+          }
+          if (!label) label = placeholder ?? 'search';
+
+          filters.push({ kind: 'search', label, locatorHint, options: [] });
+          if (filters.length >= 10) break;
+        } catch {
+          // skip
+        }
+      }
+    } catch {
+      // locator failed
+    }
+  }
+
+  return filters;
+}
+
+// ---------------------------------------------------------------------------
+// applyFilterAndCapture
+// ---------------------------------------------------------------------------
+
+async function applyFilterAndCapture(
+  page: Page,
+  filter: FilterDef,
+  option: FilterOption,
+): Promise<FilterState> {
+  const state: FilterState = {
+    filterLabel: filter.label,
+    filterValue: option.value,
+    resultCount: null,
+    firstRowValues: [],
+  };
+
+  try {
+    if (filter.kind === 'select') {
+      await page.locator(filter.locatorHint).selectOption(option.value);
+    } else if (filter.kind === 'search') {
+      await page.locator(filter.locatorHint).fill(option.value);
+      await page.locator(filter.locatorHint).press('Enter');
+    }
+
+    await waitForSettle(page, 1500);
+
+    state.resultCount = await readResultCount(page);
+    state.firstRowValues = await captureFirstRow(page);
+  } catch {
+    // non-fatal: return partial state
+  }
+
+  // Reset filter
+  try {
+    if (filter.kind === 'select') {
+      const firstOption = await page.locator(filter.locatorHint).evaluate(
+        (el: HTMLSelectElement) => el.options[0]?.value ?? '',
+      );
+      await page.locator(filter.locatorHint).selectOption(firstOption);
+    } else if (filter.kind === 'search') {
+      await page.locator(filter.locatorHint).fill('');
+      await page.locator(filter.locatorHint).press('Enter');
+    }
+    await waitForSettle(page, 800);
+  } catch {
+    // ignore reset errors
+  }
+
+  return state;
+}
+
+// ---------------------------------------------------------------------------
+// crawlFilterStates
+// ---------------------------------------------------------------------------
+
+async function crawlFilterStates(page: Page, filters: FilterDef[]): Promise<FilterState[]> {
+  const states: FilterState[] = [];
+  let totalStates = 0;
+
+  for (const filter of filters) {
+    if (totalStates >= 15) break;
+
+    const { options } = filter;
+    if (options.length === 0) continue;
+
+    // Pick first, middle, last — deduplicated, max 3
+    const indices = new Set<number>([
+      0,
+      Math.floor((options.length - 1) / 2),
+      options.length - 1,
+    ]);
+    const picked = Array.from(indices).slice(0, 3);
+
+    for (const idx of picked) {
+      if (totalStates >= 15) break;
+      const option = options[idx];
+      try {
+        const s = await applyFilterAndCapture(page, filter, option);
+        states.push(s);
+      } catch {
+        // skip
+      }
+      totalStates++;
+    }
+  }
+
+  return states;
+}
+
+// ---------------------------------------------------------------------------
+// detectPaginationKind
+// ---------------------------------------------------------------------------
+
+async function detectPaginationKind(
+  page: Page,
+): Promise<'numbered' | 'load-more' | 'infinite-scroll' | 'none'> {
+  try {
+    // Numbered pagination: ARIA page buttons or .pagination
+    const numberedCount = await page
+      .locator('[aria-label*="page" i], .pagination button, [role="navigation"] button')
+      .count()
+      .catch(() => 0);
+    if (numberedCount > 1) return 'numbered';
+
+    // Load-more button
+    const loadMore = await page
+      .locator('button:has-text("Load more"), button:has-text("Show more"), a:has-text("Load more")')
+      .count()
+      .catch(() => 0);
+    if (loadMore > 0) return 'load-more';
+
+    // Infinite scroll: scrollable content significantly taller than viewport
+    const isInfinite = await page.evaluate(() => {
+      return document.documentElement.scrollHeight > window.innerHeight * 2;
+    }).catch(() => false);
+    if (isInfinite) return 'infinite-scroll';
+  } catch {
+    // fall through
+  }
+
+  return 'none';
+}
+
+// ---------------------------------------------------------------------------
+// crawlNumberedPagination
+// ---------------------------------------------------------------------------
+
+async function crawlNumberedPagination(page: Page): Promise<PaginationCapture> {
+  const pages: Array<{ pageNum: number; rowValues: string[] }> = [];
+  const pageSizes: number[] = [];
+
+  // Read page size selector if present
+  try {
+    const sizeSelect = page.locator('select[name*="size" i], select[name*="per" i], select[aria-label*="per page" i]');
+    const sizeCount = await sizeSelect.count();
+    if (sizeCount > 0) {
+      const rawSizes = await sizeSelect.first().evaluate((el: HTMLSelectElement) =>
+        Array.from(el.options).map(o => parseInt(o.value, 10)).filter(n => !isNaN(n)),
+      );
+      pageSizes.push(...rawSizes);
+    }
+  } catch {
+    // ignore
+  }
+
+  // Determine total pages from pagination widget
+  let totalPages: number | undefined;
+  try {
+    totalPages = await page.evaluate(() => {
+      const buttons = Array.from(
+        document.querySelectorAll('[aria-label*="page" i], .pagination button'),
+      );
+      const nums = buttons
+        .map(b => parseInt((b as HTMLElement).innerText.trim(), 10))
+        .filter(n => !isNaN(n));
+      return nums.length > 0 ? Math.max(...nums) : undefined;
+    });
+  } catch {
+    // leave undefined
+  }
+
+  // Capture first page immediately
+  pages.push({ pageNum: 1, rowValues: await captureFirstRow(page) });
+
+  // Walk pages by clicking Next
+  for (let pageNum = 2; pageNum <= 20; pageNum++) {
+    try {
+      const nextBtn = page.locator(
+        'button[aria-label*="next" i], a[aria-label*="next" i], [role="navigation"] button:last-child',
+      ).first();
+
+      const isDisabled = await nextBtn.isDisabled().catch(() => true);
+      if (isDisabled) break;
+
+      await nextBtn.click();
+      await waitForSettle(page, 2000);
+
+      const rowValues = await captureFirstRow(page);
+      pages.push({ pageNum, rowValues });
+    } catch {
+      break;
+    }
+  }
+
+  return {
+    kind: 'numbered',
+    totalPages,
+    pageSizes: pageSizes.length > 0 ? pageSizes : undefined,
+    pages,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// captureInfiniteScroll
+// ---------------------------------------------------------------------------
+
+async function captureInfiniteScroll(page: Page): Promise<PaginationCapture> {
+  const pages: Array<{ pageNum: number; rowValues: string[] }> = [];
+
+  let prevRowCount = -1;
+  let iteration = 0;
+
+  while (iteration < 10) {
+    try {
+      const rowCount = await page.evaluate(() =>
+        document.querySelectorAll('tbody tr').length,
+      ).catch(() => 0);
+
+      if (rowCount === prevRowCount) break; // stable
+      prevRowCount = rowCount;
+
+      // Record current first-row snapshot per scroll step
+      pages.push({ pageNum: iteration + 1, rowValues: await captureFirstRow(page) });
+
+      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+      await new Promise<void>(r => setTimeout(r, 800));
+    } catch {
+      break;
+    }
+    iteration++;
+  }
+
+  const totalItems = await page.evaluate(() =>
+    document.querySelectorAll('tbody tr').length,
+  ).catch(() => undefined);
+
+  return {
+    kind: 'infinite-scroll',
+    totalItems: typeof totalItems === 'number' ? totalItems : undefined,
+    pages,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// captureSortColumns
+// ---------------------------------------------------------------------------
+
+async function captureSortColumns(
+  page: Page,
+): Promise<Array<{ header: string; ariaSortAfterClick: string | null }>> {
+  const results: Array<{ header: string; ariaSortAfterClick: string | null }> = [];
+
+  try {
+    const sortable = await page.locator('[aria-sort]').all();
+    for (const th of sortable.slice(0, 5)) {
+      try {
+        const header = await th.innerText().catch(() => '');
+        await th.click();
+        await waitForSettle(page, 800);
+        const ariaSortAfterClick = await th.getAttribute('aria-sort').catch(() => null);
+        results.push({ header: header.trim(), ariaSortAfterClick });
+
+        // Reset by clicking again (most tables toggle back)
+        await th.click().catch(() => undefined);
+        await waitForSettle(page, 600);
+      } catch {
+        // skip this column
+      }
+    }
+  } catch {
+    // no sortable columns found
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// crawlFiltersAndPagination — public orchestrator
+// ---------------------------------------------------------------------------
+
+export async function crawlFiltersAndPagination(page: Page): Promise<FilterPaginationResult> {
+  // 1. Discover filters
+  let filters: FilterDef[] = [];
+  try {
+    filters = await discoverFilters(page);
+  } catch {
+    // proceed with empty filters
+  }
+
+  // 2. Apply filters and capture states
+  let filterStates: FilterState[] = [];
+  try {
+    filterStates = await crawlFilterStates(page, filters);
+  } catch {
+    // proceed with empty states
+  }
+
+  // 3. Detect and crawl pagination
+  let pagination: PaginationCapture = { kind: 'none', pages: [] };
+  try {
+    const kind = await detectPaginationKind(page);
+
+    if (kind === 'numbered') {
+      pagination = await crawlNumberedPagination(page);
+    } else if (kind === 'load-more') {
+      // Treat load-more similarly to infinite scroll capture
+      pagination = await captureInfiniteScroll(page);
+      pagination = { ...pagination, kind: 'load-more' };
+    } else if (kind === 'infinite-scroll') {
+      pagination = await captureInfiniteScroll(page);
+    } else {
+      // Capture single-page rows
+      const rowValues = await captureFirstRow(page);
+      pagination = { kind: 'none', pages: rowValues.length > 0 ? [{ pageNum: 1, rowValues }] : [] };
+    }
+  } catch {
+    // leave pagination as none
+  }
+
+  // 4. Sort columns
+  let sortColumns: Array<{ header: string; ariaSortAfterClick: string | null }> = [];
+  try {
+    sortColumns = await captureSortColumns(page);
+  } catch {
+    // proceed with empty sort
+  }
+
+  return {
+    filters,
+    filterStates,
+    pagination,
+    sortColumns,
+  };
+}

package/src/gap-pass.ts

@@ -33,6 +33,29 @@ interface LlmCfg {
   provider: string;
   model: string;
   apiKey?: string;
+  baseUrl?: string;
+}
+
+// Only openai and anthropic are supported here. NVIDIA, groq, ollama, etc. → skip, fall to env.
+/**
+ * Resolve effective LLM. Priority: openai → anthropic → skip unsupported providers.
+ * Falls back to env vars when platform provider is not supported.
+ */
+function resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null {
+  if (llmCfg?.provider === 'openai' && llmCfg.apiKey) {
+    return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };
+  }
+  if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) {
+    return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model };
+  }
+  // Unsupported provider (nvidia, groq, etc.) or no key — env fallback, openai first.
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' };
+  }
+  return null;
 }
 
 /**
@@ -44,7 +67,8 @@ export async function runGapAnalysis(
   appUrl: string,
   llmCfg: LlmCfg | null | undefined,
 ): Promise<GapCandidate[]> {
-  if (!llmCfg?.apiKey || screens.length < 2) return [];
+  const effective = resolveTextLLM(llmCfg);
+  if (!effective || screens.length < 2) return [];
 
   const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();
   if (!origin) return [];
@@ -57,7 +81,8 @@ export async function runGapAnalysis(
     })
     .join('\n');
 
-  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
+  const systemPrompt = 'You are a web application coverage analyst. Respond with valid JSON only.';
+  const userPrompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
 
 Discovered screens (${screens.length} total):
 ${screenList}
@@ -70,18 +95,35 @@ Respond with ONLY valid JSON, no markdown:
 {"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`;
 
   try {
-    const Anthropic = (await import('@anthropic-ai/sdk')).default;
-    const client = new Anthropic({ apiKey: llmCfg.apiKey });
-    const response = await client.messages.create({
-      model: llmCfg.model ?? process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-haiku-4-5-20251001',
-      max_tokens: 1024,
-      temperature: 0,
-      // @ts-ignore
-      system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
-      messages: [{ role: 'user', content: prompt }],
-    });
-
-    const text = (response.content[0] as any)?.text ?? '';
+    let text = '';
+
+    if (effective.provider === 'openai') {
+      const { OpenAI } = await import('openai');
+      const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) });
+      const response = await client.chat.completions.create({
+        model: effective.model ?? 'gpt-4o-mini',
+        max_tokens: 1024,
+        temperature: 0,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      });
+      text = response.choices[0]?.message?.content ?? '';
+    } else {
+      const Anthropic = (await import('@anthropic-ai/sdk')).default;
+      const client = new Anthropic({ apiKey: effective.apiKey });
+      const response = await client.messages.create({
+        model: effective.model ?? 'claude-haiku-4-5-20251001',
+        max_tokens: 1024,
+        temperature: 0,
+        // @ts-ignore
+        system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
+        messages: [{ role: 'user', content: userPrompt }],
+      });
+      text = (response.content[0] as any)?.text ?? '';
+    }
+
     const jsonMatch = text.match(/\{[\s\S]*\}/);
     if (!jsonMatch) return [];
     const parsed = JSON.parse(jsonMatch[0]);
@@ -89,9 +131,9 @@ Respond with ONLY valid JSON, no markdown:
       parsed.gaps ?? [];
 
     if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];
-    logger.info({ gaps: rawGaps.length }, '[P6] Gap analysis: LLM returned candidates');
+    logger.info({ gaps: rawGaps.length, provider: effective.provider }, '[P6] Gap analysis: LLM returned candidates');
 
-    // HEAD-validate each URL pattern, take the first 200
+    // HEAD-validate each URL pattern
     const results: GapCandidate[] = [];
     for (const gap of rawGaps.slice(0, 15)) {
       let confirmedUrl: string | null = null;