@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/filter-pagination-crawler.ts

@@ -0,0 +1,500 @@
+/**
+ * Filter state crawling and pagination capture.
+ *
+ * Discovers filter controls (select, checkbox-group, radio-group, search),
+ * applies each meaningful option and records result counts and first-row
+ * values, then detects and walks pagination (numbered, load-more, infinite
+ * scroll). Also captures sort column behaviour.
+ *
+ * All per-filter / per-page errors are caught and skipped so one bad control
+ * never aborts the full crawl.
+ */
+
+import type { Page } from 'playwright';
+
+// ---------------------------------------------------------------------------
+// Public interfaces
+// ---------------------------------------------------------------------------
+
+export interface FilterOption {
+  value: string;
+  label: string;
+}
+
+export interface FilterDef {
+  kind: 'select' | 'checkbox-group' | 'radio-group' | 'search';
+  label: string;
+  locatorHint: string;
+  options: FilterOption[];
+}
+
+export interface FilterState {
+  filterLabel: string;
+  filterValue: string;
+  resultCount: number | null;
+  firstRowValues: string[];
+}
+
+export interface PaginationCapture {
+  kind: 'numbered' | 'load-more' | 'infinite-scroll' | 'none';
+  totalPages?: number;
+  totalItems?: number;
+  pageSizes?: number[];
+  pages: Array<{ pageNum: number; rowValues: string[] }>;
+}
+
+export interface FilterPaginationResult {
+  filters: FilterDef[];
+  filterStates: FilterState[];
+  pagination: PaginationCapture;
+  sortColumns: Array<{ header: string; ariaSortAfterClick: string | null }>;
+  searchCapture?: { noResultsText: string | null; sampleSuggestions: string[] };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/** Pause for up to `ms` ms or until network is idle, whichever comes first. */
+async function waitForSettle(page: Page, ms = 1500): Promise<void> {
+  await Promise.race([
+    page.waitForLoadState('networkidle').catch(() => undefined),
+    new Promise<void>(r => setTimeout(r, ms)),
+  ]);
+}
+
+/** Extract visible cell text from the first tbody row, if any. */
+async function captureFirstRow(page: Page): Promise<string[]> {
+  try {
+    return await page.evaluate(() => {
+      const row = document.querySelector('tbody tr');
+      if (!row) return [];
+      return Array.from(row.querySelectorAll('td')).map(td => td.innerText.trim());
+    });
+  } catch {
+    return [];
+  }
+}
+
+/** Read a result-count hint from visible page text using common patterns. */
+async function readResultCount(page: Page): Promise<number | null> {
+  try {
+    const text = await page.evaluate(() => document.body.innerText);
+    const match = text.match(/(\d[\d,]*)\s*(result|item|entr|record)/i);
+    if (!match) return null;
+    return parseInt(match[1].replace(/,/g, ''), 10);
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// discoverFilters
+// ---------------------------------------------------------------------------
+
+async function discoverFilters(page: Page): Promise<FilterDef[]> {
+  const filters: FilterDef[] = [];
+
+  // --- native <select> elements outside dialogs ---------------------------------
+  try {
+    const selectHandles = await page.locator('select:not([role="dialog"] select)').all();
+    for (const handle of selectHandles.slice(0, 10)) {
+      try {
+        const id = await handle.getAttribute('id').catch(() => null);
+        const name = await handle.getAttribute('name').catch(() => null);
+        const locatorHint = id ? `#${id}` : name ? `select[name="${name}"]` : 'select';
+
+        // Find associated label
+        let label = '';
+        if (id) {
+          label = await page.locator(`label[for="${id}"]`).innerText().catch(() => '');
+        }
+        if (!label) {
+          label = await handle.evaluate((el: HTMLSelectElement) => {
+            const lbl = el.closest('label') ?? el.previousElementSibling;
+            return (lbl as HTMLElement | null)?.innerText?.trim() ?? '';
+          }).catch(() => '');
+        }
+        if (!label) label = name ?? id ?? 'filter';
+
+        // Collect options, skipping default/placeholder entries
+        const rawOptions: FilterOption[] = await handle.evaluate((el: HTMLSelectElement) =>
+          Array.from(el.options).map(o => ({ value: o.value, label: o.text.trim() })),
+        ).catch(() => []);
+
+        const options = rawOptions.filter(
+          o => !(/^all$|^any$|^select|^--/i.test(o.label) || o.value === ''),
+        );
+
+        if (options.length === 0) continue;
+
+        filters.push({ kind: 'select', label, locatorHint, options });
+        if (filters.length >= 10) break;
+      } catch {
+        // skip this select
+      }
+    }
+  } catch {
+    // page.locator failed
+  }
+
+  // --- search inputs ------------------------------------------------------------
+  if (filters.length < 10) {
+    try {
+      const searchHandles = await page
+        .locator(
+          'input[type="search"]:not([role="dialog"] input), ' +
+          'input[placeholder*="search" i]:not([role="dialog"] input)',
+        )
+        .all();
+
+      for (const handle of searchHandles.slice(0, 10 - filters.length)) {
+        try {
+          const id = await handle.getAttribute('id').catch(() => null);
+          const placeholder = await handle.getAttribute('placeholder').catch(() => null);
+          const locatorHint = id ? `#${id}` : placeholder ? `input[placeholder="${placeholder}"]` : 'input[type="search"]';
+
+          let label = '';
+          if (id) {
+            label = await page.locator(`label[for="${id}"]`).innerText().catch(() => '');
+          }
+          if (!label) label = placeholder ?? 'search';
+
+          filters.push({ kind: 'search', label, locatorHint, options: [] });
+          if (filters.length >= 10) break;
+        } catch {
+          // skip
+        }
+      }
+    } catch {
+      // locator failed
+    }
+  }
+
+  return filters;
+}
+
+// ---------------------------------------------------------------------------
+// applyFilterAndCapture
+// ---------------------------------------------------------------------------
+
+async function applyFilterAndCapture(
+  page: Page,
+  filter: FilterDef,
+  option: FilterOption,
+): Promise<FilterState> {
+  const state: FilterState = {
+    filterLabel: filter.label,
+    filterValue: option.value,
+    resultCount: null,
+    firstRowValues: [],
+  };
+
+  try {
+    if (filter.kind === 'select') {
+      await page.locator(filter.locatorHint).selectOption(option.value);
+    } else if (filter.kind === 'search') {
+      await page.locator(filter.locatorHint).fill(option.value);
+      await page.locator(filter.locatorHint).press('Enter');
+    }
+
+    await waitForSettle(page, 1500);
+
+    state.resultCount = await readResultCount(page);
+    state.firstRowValues = await captureFirstRow(page);
+  } catch {
+    // non-fatal: return partial state
+  }
+
+  // Reset filter
+  try {
+    if (filter.kind === 'select') {
+      const firstOption = await page.locator(filter.locatorHint).evaluate(
+        (el: HTMLSelectElement) => el.options[0]?.value ?? '',
+      );
+      await page.locator(filter.locatorHint).selectOption(firstOption);
+    } else if (filter.kind === 'search') {
+      await page.locator(filter.locatorHint).fill('');
+      await page.locator(filter.locatorHint).press('Enter');
+    }
+    await waitForSettle(page, 800);
+  } catch {
+    // ignore reset errors
+  }
+
+  return state;
+}
+
+// ---------------------------------------------------------------------------
+// crawlFilterStates
+// ---------------------------------------------------------------------------
+
+async function crawlFilterStates(page: Page, filters: FilterDef[]): Promise<FilterState[]> {
+  const states: FilterState[] = [];
+  let totalStates = 0;
+
+  for (const filter of filters) {
+    if (totalStates >= 15) break;
+
+    const { options } = filter;
+    if (options.length === 0) continue;
+
+    // Pick first, middle, last — deduplicated, max 3
+    const indices = new Set<number>([
+      0,
+      Math.floor((options.length - 1) / 2),
+      options.length - 1,
+    ]);
+    const picked = Array.from(indices).slice(0, 3);
+
+    for (const idx of picked) {
+      if (totalStates >= 15) break;
+      const option = options[idx];
+      try {
+        const s = await applyFilterAndCapture(page, filter, option);
+        states.push(s);
+      } catch {
+        // skip
+      }
+      totalStates++;
+    }
+  }
+
+  return states;
+}
+
+// ---------------------------------------------------------------------------
+// detectPaginationKind
+// ---------------------------------------------------------------------------
+
+async function detectPaginationKind(
+  page: Page,
+): Promise<'numbered' | 'load-more' | 'infinite-scroll' | 'none'> {
+  try {
+    // Numbered pagination: ARIA page buttons or .pagination
+    const numberedCount = await page
+      .locator('[aria-label*="page" i], .pagination button, [role="navigation"] button')
+      .count()
+      .catch(() => 0);
+    if (numberedCount > 1) return 'numbered';
+
+    // Load-more button
+    const loadMore = await page
+      .locator('button:has-text("Load more"), button:has-text("Show more"), a:has-text("Load more")')
+      .count()
+      .catch(() => 0);
+    if (loadMore > 0) return 'load-more';
+
+    // Infinite scroll: scrollable content significantly taller than viewport
+    const isInfinite = await page.evaluate(() => {
+      return document.documentElement.scrollHeight > window.innerHeight * 2;
+    }).catch(() => false);
+    if (isInfinite) return 'infinite-scroll';
+  } catch {
+    // fall through
+  }
+
+  return 'none';
+}
+
+// ---------------------------------------------------------------------------
+// crawlNumberedPagination
+// ---------------------------------------------------------------------------
+
+async function crawlNumberedPagination(page: Page): Promise<PaginationCapture> {
+  const pages: Array<{ pageNum: number; rowValues: string[] }> = [];
+  const pageSizes: number[] = [];
+
+  // Read page size selector if present
+  try {
+    const sizeSelect = page.locator('select[name*="size" i], select[name*="per" i], select[aria-label*="per page" i]');
+    const sizeCount = await sizeSelect.count();
+    if (sizeCount > 0) {
+      const rawSizes = await sizeSelect.first().evaluate((el: HTMLSelectElement) =>
+        Array.from(el.options).map(o => parseInt(o.value, 10)).filter(n => !isNaN(n)),
+      );
+      pageSizes.push(...rawSizes);
+    }
+  } catch {
+    // ignore
+  }
+
+  // Determine total pages from pagination widget
+  let totalPages: number | undefined;
+  try {
+    totalPages = await page.evaluate(() => {
+      const buttons = Array.from(
+        document.querySelectorAll('[aria-label*="page" i], .pagination button'),
+      );
+      const nums = buttons
+        .map(b => parseInt((b as HTMLElement).innerText.trim(), 10))
+        .filter(n => !isNaN(n));
+      return nums.length > 0 ? Math.max(...nums) : undefined;
+    });
+  } catch {
+    // leave undefined
+  }
+
+  // Capture first page immediately
+  pages.push({ pageNum: 1, rowValues: await captureFirstRow(page) });
+
+  // Walk pages by clicking Next
+  for (let pageNum = 2; pageNum <= 20; pageNum++) {
+    try {
+      const nextBtn = page.locator(
+        'button[aria-label*="next" i], a[aria-label*="next" i], [role="navigation"] button:last-child',
+      ).first();
+
+      const isDisabled = await nextBtn.isDisabled().catch(() => true);
+      if (isDisabled) break;
+
+      await nextBtn.click();
+      await waitForSettle(page, 2000);
+
+      const rowValues = await captureFirstRow(page);
+      pages.push({ pageNum, rowValues });
+    } catch {
+      break;
+    }
+  }
+
+  return {
+    kind: 'numbered',
+    totalPages,
+    pageSizes: pageSizes.length > 0 ? pageSizes : undefined,
+    pages,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// captureInfiniteScroll
+// ---------------------------------------------------------------------------
+
+async function captureInfiniteScroll(page: Page): Promise<PaginationCapture> {
+  const pages: Array<{ pageNum: number; rowValues: string[] }> = [];
+
+  let prevRowCount = -1;
+  let iteration = 0;
+
+  while (iteration < 10) {
+    try {
+      const rowCount = await page.evaluate(() =>
+        document.querySelectorAll('tbody tr').length,
+      ).catch(() => 0);
+
+      if (rowCount === prevRowCount) break; // stable
+      prevRowCount = rowCount;
+
+      // Record current first-row snapshot per scroll step
+      pages.push({ pageNum: iteration + 1, rowValues: await captureFirstRow(page) });
+
+      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+      await new Promise<void>(r => setTimeout(r, 800));
+    } catch {
+      break;
+    }
+    iteration++;
+  }
+
+  const totalItems = await page.evaluate(() =>
+    document.querySelectorAll('tbody tr').length,
+  ).catch(() => undefined);
+
+  return {
+    kind: 'infinite-scroll',
+    totalItems: typeof totalItems === 'number' ? totalItems : undefined,
+    pages,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// captureSortColumns
+// ---------------------------------------------------------------------------
+
+async function captureSortColumns(
+  page: Page,
+): Promise<Array<{ header: string; ariaSortAfterClick: string | null }>> {
+  const results: Array<{ header: string; ariaSortAfterClick: string | null }> = [];
+
+  try {
+    const sortable = await page.locator('[aria-sort]').all();
+    for (const th of sortable.slice(0, 5)) {
+      try {
+        const header = await th.innerText().catch(() => '');
+        await th.click();
+        await waitForSettle(page, 800);
+        const ariaSortAfterClick = await th.getAttribute('aria-sort').catch(() => null);
+        results.push({ header: header.trim(), ariaSortAfterClick });
+
+        // Reset by clicking again (most tables toggle back)
+        await th.click().catch(() => undefined);
+        await waitForSettle(page, 600);
+      } catch {
+        // skip this column
+      }
+    }
+  } catch {
+    // no sortable columns found
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// crawlFiltersAndPagination — public orchestrator
+// ---------------------------------------------------------------------------
+
+export async function crawlFiltersAndPagination(page: Page): Promise<FilterPaginationResult> {
+  // 1. Discover filters
+  let filters: FilterDef[] = [];
+  try {
+    filters = await discoverFilters(page);
+  } catch {
+    // proceed with empty filters
+  }
+
+  // 2. Apply filters and capture states
+  let filterStates: FilterState[] = [];
+  try {
+    filterStates = await crawlFilterStates(page, filters);
+  } catch {
+    // proceed with empty states
+  }
+
+  // 3. Detect and crawl pagination
+  let pagination: PaginationCapture = { kind: 'none', pages: [] };
+  try {
+    const kind = await detectPaginationKind(page);
+
+    if (kind === 'numbered') {
+      pagination = await crawlNumberedPagination(page);
+    } else if (kind === 'load-more') {
+      // Treat load-more similarly to infinite scroll capture
+      pagination = await captureInfiniteScroll(page);
+      pagination = { ...pagination, kind: 'load-more' };
+    } else if (kind === 'infinite-scroll') {
+      pagination = await captureInfiniteScroll(page);
+    } else {
+      // Capture single-page rows
+      const rowValues = await captureFirstRow(page);
+      pagination = { kind: 'none', pages: rowValues.length > 0 ? [{ pageNum: 1, rowValues }] : [] };
+    }
+  } catch {
+    // leave pagination as none
+  }
+
+  // 4. Sort columns
+  let sortColumns: Array<{ header: string; ariaSortAfterClick: string | null }> = [];
+  try {
+    sortColumns = await captureSortColumns(page);
+  } catch {
+    // proceed with empty sort
+  }
+
+  return {
+    filters,
+    filterStates,
+    pagination,
+    sortColumns,
+  };
+}

package/src/form-detector.ts

@@ -0,0 +1,103 @@
+/**
+ * Form detector — extract form metadata from crawled pages.
+ *
+ * For each form found on a page, captures: action URL, method, all input fields
+ * with their types/names/required/validation constraints, and submit button text.
+ *
+ * This data feeds ZeTa's test generator to create form validation test cases
+ * automatically (happy path + boundary/error cases per field type).
+ */
+
+import type { Page } from 'playwright';
+
+export interface FormField {
+  name: string;
+  type: string;        // text, email, password, number, select, textarea, checkbox, radio, file, date
+  label?: string;
+  required: boolean;
+  placeholder?: string;
+  minLength?: number;
+  maxLength?: number;
+  min?: string;
+  max?: string;
+  pattern?: string;
+  options?: string[]; // for select/radio
+  autocomplete?: string;
+}
+
+export interface DetectedForm {
+  index: number;
+  action?: string;
+  method: string;
+  fields: FormField[];
+  submitText?: string;
+  hasFileUpload: boolean;
+  purpose: 'login' | 'search' | 'registration' | 'contact' | 'checkout' | 'settings' | 'unknown';
+}
+
+function guessPurpose(fields: FormField[], action?: string): DetectedForm['purpose'] {
+  const names = fields.map(f => f.name?.toLowerCase() + (f.label ?? '').toLowerCase()).join(' ');
+  if (/password/.test(names) && /email|username|user/.test(names) && fields.length <= 4) return 'login';
+  if (/password/.test(names) && /confirm|repeat/.test(names)) return 'registration';
+  if (/search|query|q\b/.test(names) && fields.length <= 3) return 'search';
+  if (/card|payment|cvv|expir/.test(names)) return 'checkout';
+  if (/message|subject|contact/.test(names)) return 'contact';
+  if (/setting|preference|profile/.test(names)) return 'settings';
+  return 'unknown';
+}
+
+export async function detectForms(page: Page): Promise<DetectedForm[]> {
+  try {
+    return await page.$$eval('form', (forms) => forms.map((form: any, index: number) => {
+      const inputs = Array.from(form.querySelectorAll('input,select,textarea,button[type="submit"]'));
+      const fields: any[] = [];
+
+      for (const input of inputs as any[]) {
+        const tag = input.tagName.toLowerCase();
+        const type = tag === 'button' ? 'submit' : (input.type || tag);
+        if (['submit', 'reset', 'button', 'hidden'].includes(type)) continue;
+
+        // Find associated label
+        let label = '';
+        if (input.id) {
+          const lbl = form.querySelector(`label[for="${input.id}"]`);
+          if (lbl) label = (lbl.textContent ?? '').trim();
+        }
+        if (!label) {
+          const parent = input.closest('label');
+          if (parent) label = (parent.textContent ?? '').replace(input.value ?? '', '').trim().slice(0, 80);
+        }
+
+        const options = tag === 'select'
+          ? Array.from(input.options ?? []).map((o: any) => o.text?.trim()).filter(Boolean).slice(0, 20)
+          : tag === 'input' && type === 'radio'
+          ? [input.value]
+          : undefined;
+
+        fields.push({
+          name: input.name || input.id || '',
+          type,
+          label: label || undefined,
+          required: !!input.required,
+          placeholder: input.placeholder || undefined,
+          minLength: input.minLength > 0 ? input.minLength : undefined,
+          maxLength: input.maxLength > 0 && input.maxLength < 524288 ? input.maxLength : undefined,
+          min: input.min || undefined,
+          max: input.max || undefined,
+          pattern: input.pattern || undefined,
+          options,
+          autocomplete: input.autocomplete || undefined,
+        });
+      }
+
+      const submitBtn = form.querySelector('button[type="submit"],input[type="submit"]') as any;
+      const hasFileUpload = !!form.querySelector('input[type="file"]');
+      const action = form.action || undefined;
+      const method = (form.method || 'get').toUpperCase();
+
+      return { index, action, method, fields, submitText: submitBtn?.textContent?.trim() || submitBtn?.value || undefined, hasFileUpload };
+    })).then(forms => forms.map((f: any) => ({ ...f, purpose: guessPurpose(f.fields, f.action) })));
+  } catch {
+    return [];
+  }
+}

package/src/frontier.test.ts

@@ -0,0 +1,64 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { CrawlFrontier } from './frontier.js';
+
+test('CrawlFrontier: BFS dequeues in FIFO order, identical to a plain array shift()', () => {
+  const f = new CrawlFrontier('BFS', ['https://a.com/1', 'https://a.com/2']);
+  f.push('https://a.com/3');
+  assert.equal(f.shift(), 'https://a.com/1');
+  assert.equal(f.shift(), 'https://a.com/2');
+  assert.equal(f.shift(), 'https://a.com/3');
+  assert.equal(f.shift(), undefined);
+});
+
+test('CrawlFrontier: DFS dequeues in LIFO order', () => {
+  const f = new CrawlFrontier('DFS', ['https://a.com/1', 'https://a.com/2']);
+  f.push('https://a.com/3');
+  assert.equal(f.shift(), 'https://a.com/3');
+  assert.equal(f.shift(), 'https://a.com/2');
+  assert.equal(f.shift(), 'https://a.com/1');
+});
+
+test('CrawlFrontier: BEST_FIRST prefers shallower paths over deeper ones', () => {
+  const f = new CrawlFrontier('BEST_FIRST', [
+    'https://a.com/products/electronics/phones/model-x',
+    'https://a.com/about',
+  ]);
+  assert.equal(f.shift(), 'https://a.com/about');
+  assert.equal(f.shift(), 'https://a.com/products/electronics/phones/model-x');
+});
+
+test('CrawlFrontier: BEST_FIRST boosts URLs matching keywords ahead of shallower non-matching ones', () => {
+  const f = new CrawlFrontier('BEST_FIRST', ['https://a.com/'], ['pricing']);
+  f.push('https://a.com/deep/nested/pricing/plans');
+  // "/" has depth 0 (score 0); the pricing URL has depth 4 (score -4) but +10 for the
+  // keyword match, netting +6 — the keyword-matching page should win despite depth.
+  assert.equal(f.shift(), 'https://a.com/deep/nested/pricing/plans');
+  assert.equal(f.shift(), 'https://a.com/');
+});
+
+test('CrawlFrontier: length reflects push/shift correctly across all strategies', () => {
+  for (const strategy of ['BFS', 'DFS', 'BEST_FIRST'] as const) {
+    const f = new CrawlFrontier(strategy, ['https://a.com/1']);
+    assert.equal(f.length, 1);
+    f.push('https://a.com/2');
+    assert.equal(f.length, 2);
+    f.shift();
+    assert.equal(f.length, 1);
+  }
+});
+
+test('CrawlFrontier: filter returns URLs matching the predicate without mutating the frontier', () => {
+  const f = new CrawlFrontier('BFS', ['https://a.com/1', 'https://a.com/2', 'https://a.com/3']);
+  const matched = f.filter((u) => u.endsWith('/2'));
+  assert.deepEqual(matched, ['https://a.com/2']);
+  assert.equal(f.length, 3); // unchanged
+});
+
+test('CrawlFrontier: malformed URLs never throw when scored under BEST_FIRST', () => {
+  assert.doesNotThrow(() => {
+    const f = new CrawlFrontier('BEST_FIRST', ['not-a-valid-url', 'https://a.com/ok']);
+    f.shift();
+    f.shift();
+  });
+});