@detiq/crawler 1.1.13 → 1.1.15

package/src/advanced-ui-probes.ts

@@ -0,0 +1,470 @@
+/**
+ * Advanced UI probes — detect and exercise drag-drop, date pickers, sliders,
+ * and typeahead/autocomplete widgets on a crawled page.
+ *
+ * Results feed ZeTa's test generator so it can emit interaction-level test
+ * cases for these widget families instead of falling back to generic clicks.
+ *
+ * Each probe is independently try/caught so a broken widget does not abort
+ * the others. All browser-side code runs inside page.evaluate / page.$$eval
+ * to avoid serialisation issues with DOM handles.
+ */
+
+import type { Page } from 'playwright';
+
+// ---------------------------------------------------------------------------
+// Public interfaces
+// ---------------------------------------------------------------------------
+
+export interface DragDropCapture {
+  library: 'html5' | 'dnd-kit' | 'react-beautiful-dnd' | 'sortablejs' | 'pointer-event' | 'none';
+  draggableCount: number;
+  dropZoneCount: number;
+  testAttempted: boolean;
+  testCompleted: boolean;
+}
+
+export interface DatePickerCapture {
+  fieldLabel: string;
+  pickerType: 'calendar-popup' | 'native-date' | 'native-datetime' | 'none';
+  minDate?: string;
+  maxDate?: string;
+  disabledDatesFound: boolean;
+  sampleSelectedValue?: string;
+}
+
+export interface SliderCapture {
+  label: string;
+  min: number;
+  max: number;
+  step: number;
+  currentValue: number;
+}
+
+export interface TypeaheadCapture {
+  label: string;
+  minCharsToTrigger: number;
+  sampleSuggestions: string[];
+  noResultsText: string | null;
+}
+
+export interface AdvancedProbeResult {
+  dragDrop: DragDropCapture;
+  datePickers: DatePickerCapture[];
+  sliders: SliderCapture[];
+  typeaheads: TypeaheadCapture[];
+}
+
+// ---------------------------------------------------------------------------
+// Internal: drag-and-drop detection
+// ---------------------------------------------------------------------------
+
+async function detectDragAndDrop(page: Page): Promise<DragDropCapture> {
+  // Detect library and count elements in the browser context
+  const detected = await page.evaluate((): {
+    library: DragDropCapture['library'];
+    draggableCount: number;
+    dropZoneCount: number;
+  } => {
+    // dnd-kit
+    if (document.querySelector('[data-dnd-kit-draggable]')) {
+      return {
+        library: 'dnd-kit',
+        draggableCount: document.querySelectorAll('[data-dnd-kit-draggable]').length,
+        dropZoneCount: document.querySelectorAll('[data-dnd-kit-droppable]').length,
+      };
+    }
+    // react-beautiful-dnd
+    if (document.querySelector('[data-rbd-draggable-id]')) {
+      return {
+        library: 'react-beautiful-dnd',
+        draggableCount: document.querySelectorAll('[data-rbd-draggable-id]').length,
+        dropZoneCount: document.querySelectorAll('[data-rbd-droppable-id]').length,
+      };
+    }
+    // SortableJS
+    if (document.querySelector('.sortable-ghost') || document.querySelector('[data-sortable]')) {
+      return {
+        library: 'sortablejs',
+        draggableCount: document.querySelectorAll('[data-sortable], .sortable-item').length,
+        dropZoneCount: document.querySelectorAll('[data-sortable-list], .sortable-list').length,
+      };
+    }
+    // Native HTML5 draggable
+    const html5Draggables = document.querySelectorAll('[draggable="true"]');
+    if (html5Draggables.length > 0) {
+      return {
+        library: 'html5',
+        draggableCount: html5Draggables.length,
+        dropZoneCount: document.querySelectorAll('[data-drop-target], [ondrop], .drop-zone').length,
+      };
+    }
+    return { library: 'none', draggableCount: 0, dropZoneCount: 0 };
+  });
+
+  if (detected.library === 'none') {
+    return { ...detected, testAttempted: false, testCompleted: false };
+  }
+
+  let testAttempted = false;
+  let testCompleted = false;
+
+  try {
+    if (detected.library === 'html5' && detected.draggableCount >= 2) {
+      // Attempt a real dragAndDrop between the first two draggable elements
+      const sources = await page.$$('[draggable="true"]');
+      if (sources.length >= 2) {
+        testAttempted = true;
+        await page.dragAndDrop('[draggable="true"]:nth-child(1)', '[draggable="true"]:nth-child(2)');
+        testCompleted = true;
+      }
+    } else if (detected.draggableCount >= 2) {
+      // Pointer-event simulation for library-managed drag
+      const draggableSel =
+        detected.library === 'dnd-kit' ? '[data-dnd-kit-draggable]' :
+        detected.library === 'react-beautiful-dnd' ? '[data-rbd-draggable-id]' :
+        '[data-sortable], .sortable-item';
+
+      const elements = await page.$$(draggableSel);
+      if (elements.length >= 2) {
+        testAttempted = true;
+        const src = elements[0];
+        const dst = elements[1];
+        const srcBox = await src.boundingBox();
+        const dstBox = await dst.boundingBox();
+
+        if (srcBox && dstBox) {
+          const srcX = srcBox.x + srcBox.width / 2;
+          const srcY = srcBox.y + srcBox.height / 2;
+          const dstX = dstBox.x + dstBox.width / 2;
+          const dstY = dstBox.y + dstBox.height / 2;
+
+          await page.mouse.move(srcX, srcY);
+          await page.mouse.down();
+          // Move in small steps to allow drag handlers to fire
+          const steps = 10;
+          for (let i = 1; i <= steps; i++) {
+            await page.mouse.move(
+              srcX + ((dstX - srcX) * i) / steps,
+              srcY + ((dstY - srcY) * i) / steps,
+            );
+          }
+          await page.mouse.up();
+          testCompleted = true;
+        }
+      }
+    }
+  } catch {
+    // testCompleted remains false — that is the signal to the caller
+  }
+
+  return { ...detected, testAttempted, testCompleted };
+}
+
+// ---------------------------------------------------------------------------
+// Internal: date pickers
+// ---------------------------------------------------------------------------
+
+async function captureDatePickers(page: Page): Promise<DatePickerCapture[]> {
+  const results: DatePickerCapture[] = [];
+
+  // Native date / datetime-local inputs
+  const nativeInputs = await page.$$('input[type="date"], input[type="datetime-local"]');
+  for (const input of nativeInputs.slice(0, 5)) {
+    try {
+      const attrs = await input.evaluate((el: any) => ({
+        type: el.type as string,
+        min: el.min || undefined,
+        max: el.max || undefined,
+        id: el.id || '',
+        name: el.name || '',
+      }));
+
+      // Resolve label text
+      let fieldLabel = attrs.id || attrs.name || 'unknown';
+      if (attrs.id) {
+        const lblText = await page.evaluate((id: string) => {
+          const lbl = document.querySelector(`label[for="${id}"]`);
+          return lbl ? (lbl.textContent ?? '').trim() : null;
+        }, attrs.id);
+        if (lblText) fieldLabel = lblText;
+      }
+
+      // Try filling a known date and reading it back
+      let sampleSelectedValue: string | undefined;
+      try {
+        await input.fill('2026-09-15');
+        sampleSelectedValue = await input.inputValue();
+        if (!sampleSelectedValue) sampleSelectedValue = undefined;
+      } catch {
+        // ignore fill errors
+      }
+
+      results.push({
+        fieldLabel,
+        pickerType: attrs.type === 'datetime-local' ? 'native-datetime' : 'native-date',
+        minDate: attrs.min,
+        maxDate: attrs.max,
+        disabledDatesFound: false, // native inputs have no visual disabled cells
+        sampleSelectedValue,
+      });
+    } catch {
+      // skip this element
+    }
+  }
+
+  // Calendar-popup pickers (custom widgets triggered by click)
+  // Only look for inputs not already classified as native date inputs
+  const customInputs = await page.$$('input:not([type="date"]):not([type="datetime-local"]):not([type="hidden"])');
+  for (const input of customInputs.slice(0, 5)) {
+    if (results.length >= 5) break;
+
+    try {
+      await input.click();
+      // Wait up to 2 s for a calendar grid to appear
+      const calendarAppeared = await page.waitForSelector('[role="grid"]', { timeout: 2000 })
+        .then(() => true)
+        .catch(() => false);
+
+      if (!calendarAppeared) {
+        await page.keyboard.press('Escape');
+        continue;
+      }
+
+      // Capture month/year header
+      const monthHeader = await page.evaluate(() => {
+        const header = document.querySelector('[role="grid"] caption, [role="columnheader"], .calendar-header, .rdp-caption');
+        return header ? (header.textContent ?? '').trim().slice(0, 60) : '';
+      });
+
+      // Count disabled cells
+      const disabledCount = await page.evaluate(() =>
+        document.querySelectorAll('[role="gridcell"][aria-disabled="true"], [role="gridcell"].disabled').length
+      );
+
+      // Resolve label for this input
+      const inputId = await input.evaluate((el: any) => el.id || el.name || '');
+      let fieldLabel = inputId || 'datepicker';
+      if (inputId) {
+        const lblText = await page.evaluate((id: string) => {
+          const lbl = document.querySelector(`label[for="${id}"]`);
+          return lbl ? (lbl.textContent ?? '').trim() : null;
+        }, inputId);
+        if (lblText) fieldLabel = lblText;
+      }
+
+      // Click the first enabled cell and read value
+      let sampleSelectedValue: string | undefined;
+      try {
+        const firstEnabled = await page.$('[role="gridcell"]:not([aria-disabled="true"]):not(.disabled)');
+        if (firstEnabled) {
+          await firstEnabled.click();
+          sampleSelectedValue = await input.inputValue().catch(() => undefined);
+        }
+      } catch {
+        // ignore
+      }
+
+      await page.keyboard.press('Escape');
+
+      results.push({
+        fieldLabel,
+        pickerType: 'calendar-popup',
+        disabledDatesFound: disabledCount > 0,
+        sampleSelectedValue: sampleSelectedValue || undefined,
+      });
+    } catch {
+      try { await page.keyboard.press('Escape'); } catch { /* ignore */ }
+    }
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Internal: sliders
+// ---------------------------------------------------------------------------
+
+async function captureSliders(page: Page): Promise<SliderCapture[]> {
+  const results: SliderCapture[] = [];
+
+  // Collect both native range inputs and ARIA sliders
+  const handles = await page.$$('input[type="range"], [role="slider"]');
+
+  for (const handle of handles.slice(0, 5)) {
+    try {
+      const attrs = await handle.evaluate((el: any) => {
+        const min = parseFloat(el.min ?? el.getAttribute('aria-valuemin') ?? '0');
+        const max = parseFloat(el.max ?? el.getAttribute('aria-valuemax') ?? '100');
+        const step = parseFloat(el.step ?? el.getAttribute('aria-valuestep') ?? '1') || 1;
+        const value = parseFloat(el.value ?? el.getAttribute('aria-valuenow') ?? String(min));
+        const id = el.id || el.name || '';
+        return { min, max, step, value, id };
+      });
+
+      // Resolve label
+      let label = attrs.id || 'slider';
+      if (attrs.id) {
+        const lblText = await page.evaluate((id: string) => {
+          const lbl = document.querySelector(`label[for="${id}"]`);
+          return lbl ? (lbl.textContent ?? '').trim() : null;
+        }, attrs.id);
+        if (lblText) label = lblText;
+      }
+
+      // Exercise fill to min then max then restore (native range only)
+      const isNative = await handle.evaluate((el: any) => el.tagName.toLowerCase() === 'input');
+      if (isNative) {
+        try {
+          await handle.fill(String(attrs.min));
+          await handle.fill(String(attrs.max));
+          await handle.fill(String(attrs.value));
+        } catch {
+          // ignore — read-only or detached
+        }
+      }
+
+      results.push({
+        label,
+        min: attrs.min,
+        max: attrs.max,
+        step: attrs.step,
+        currentValue: attrs.value,
+      });
+    } catch {
+      // skip this element
+    }
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Internal: typeahead / autocomplete
+// ---------------------------------------------------------------------------
+
+async function captureTypeaheads(page: Page): Promise<TypeaheadCapture[]> {
+  const results: TypeaheadCapture[] = [];
+
+  // Candidate selectors for typeahead inputs
+  const candidates = await page.$$(
+    '[role="combobox"][aria-autocomplete], [aria-autocomplete="list"], input.autocomplete, input[aria-autocomplete]'
+  );
+
+  for (const input of candidates.slice(0, 3)) {
+    try {
+      const inputId = await input.evaluate((el: any) => el.id || el.name || el.placeholder || '');
+
+      // Resolve label
+      let label = inputId || 'typeahead';
+      if (inputId) {
+        const lblText = await page.evaluate((id: string) => {
+          const lbl = document.querySelector(`label[for="${id}"]`);
+          return lbl ? (lbl.textContent ?? '').trim() : null;
+        }, inputId);
+        if (lblText) label = lblText;
+      }
+
+      await input.click();
+      await input.fill('');
+
+      // Type 'a' and check whether a listbox appears
+      await input.type('a', { delay: 100 });
+      await page.waitForTimeout(500);
+
+      let listboxVisible = await page.evaluate(() => {
+        const lb = document.querySelector('[role="listbox"]');
+        if (!lb) return false;
+        const style = window.getComputedStyle(lb);
+        return style.display !== 'none' && style.visibility !== 'hidden';
+      });
+
+      let minCharsToTrigger = listboxVisible ? 1 : 2;
+
+      if (!listboxVisible) {
+        // Try a second character
+        await input.type('b', { delay: 100 });
+        await page.waitForTimeout(500);
+        listboxVisible = await page.evaluate(() => {
+          const lb = document.querySelector('[role="listbox"]');
+          if (!lb) return false;
+          const style = window.getComputedStyle(lb);
+          return style.display !== 'none' && style.visibility !== 'hidden';
+        });
+        if (listboxVisible) minCharsToTrigger = 2;
+      }
+
+      // Collect suggestion text from visible options
+      const sampleSuggestions: string[] = [];
+      if (listboxVisible) {
+        const suggestions = await page.evaluate(() =>
+          Array.from(document.querySelectorAll('[role="option"]'))
+            .slice(0, 8)
+            .map(o => (o.textContent ?? '').trim())
+            .filter(Boolean)
+        );
+        sampleSuggestions.push(...suggestions);
+      }
+
+      // Type a nonsense string to capture the "no results" message
+      await input.fill('ZZXQQ_NORESULT_9999');
+      await page.waitForTimeout(500);
+
+      const noResultsText = await page.evaluate(() => {
+        // Common no-results patterns
+        const selectors = [
+          '[role="listbox"] [role="option"]',
+          '[role="status"]',
+          '.no-results',
+          '.empty-message',
+          '[data-testid*="no-result"]',
+        ];
+        for (const sel of selectors) {
+          const el = document.querySelector(sel);
+          if (el) {
+            const txt = (el.textContent ?? '').trim();
+            if (txt) return txt.slice(0, 120);
+          }
+        }
+        return null;
+      });
+
+      // Clear the input to leave the page in a clean state
+      await input.fill('');
+
+      results.push({
+        label,
+        minCharsToTrigger,
+        sampleSuggestions,
+        noResultsText,
+      });
+    } catch {
+      try { await input.fill(''); } catch { /* ignore */ }
+    }
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Public orchestrator
+// ---------------------------------------------------------------------------
+
+export async function probeAdvancedUI(page: Page): Promise<AdvancedProbeResult> {
+  const defaultDragDrop: DragDropCapture = {
+    library: 'none',
+    draggableCount: 0,
+    dropZoneCount: 0,
+    testAttempted: false,
+    testCompleted: false,
+  };
+
+  const [dragDrop, datePickers, sliders, typeaheads] = await Promise.all([
+    detectDragAndDrop(page).catch((): DragDropCapture => defaultDragDrop),
+    captureDatePickers(page).catch((): DatePickerCapture[] => []),
+    captureSliders(page).catch((): SliderCapture[] => []),
+    captureTypeaheads(page).catch((): TypeaheadCapture[] => []),
+  ]);
+
+  return { dragDrop, datePickers, sliders, typeaheads };
+}

package/src/crawl-session-manager.ts

@@ -0,0 +1,231 @@
+/**
+ * Session management for authenticated crawl sessions.
+ * Detects auth expiry via HTTP status codes and login-page redirects,
+ * then attempts recovery via a token refresh endpoint or re-login flow.
+ * Designed to keep long-running crawls alive without manual intervention.
+ */
+
+import type { Page } from 'playwright';
+
+export interface SessionConfig {
+  loginUrl?: string;
+  email?: string;
+  password?: string;
+  healthCheckUrl?: string;
+  refreshEndpoint?: string;
+  tokenLocalStorageKey?: string;
+  maxRefreshAttempts?: number;
+}
+
+export type SessionEventType =
+  | 'HEALTHY'
+  | 'EXPIRY_DETECTED'
+  | 'REFRESH_ATTEMPT'
+  | 'REFRESH_SUCCESS'
+  | 'REFRESH_FAILED'
+  | 'AUTH_FAILURE';
+
+export interface SessionEvent {
+  type: SessionEventType;
+  isoTimestamp: string;
+  url?: string;
+  strategy?: string;
+  attemptCount?: number;
+}
+
+export interface SessionManager {
+  onResponse(url: string, status: number): void;
+  onNavigation(url: string): void;
+  isHealthy(): boolean;
+  needsRefresh(): boolean;
+  markHealthy(isoTimestamp: string): void;
+  refresh(page: Page, isoTimestamp: string): Promise<boolean>;
+  getEvents(): SessionEvent[];
+  getStats(): { healthy: boolean; refreshAttempts: number; authFailure: boolean };
+}
+
+const LOGIN_PATH_PATTERNS = [
+  '/login',
+  '/sign-in',
+  '/signin',
+  '/auth/login',
+  '/auth/sign-in',
+];
+
+export function createSessionManager(config: SessionConfig): SessionManager {
+  let healthy = true;
+  let refreshAttempts = 0;
+  let authFailure = false;
+  const events: SessionEvent[] = [];
+  const maxAttempts = config.maxRefreshAttempts ?? 3;
+
+  function onResponse(url: string, status: number): void {
+    if ((status === 401 || status === 403) && healthy) {
+      healthy = false;
+      events.push({ type: 'EXPIRY_DETECTED', isoTimestamp: new Date().toISOString(), url });
+    }
+  }
+
+  function onNavigation(url: string): void {
+    const isLoginPage = LOGIN_PATH_PATTERNS.some((pattern) => url.includes(pattern));
+    if (isLoginPage && healthy) {
+      healthy = false;
+      events.push({ type: 'EXPIRY_DETECTED', isoTimestamp: new Date().toISOString(), url });
+    }
+  }
+
+  function isHealthy(): boolean {
+    return healthy;
+  }
+
+  function needsRefresh(): boolean {
+    return !healthy && !authFailure && refreshAttempts < maxAttempts;
+  }
+
+  function markHealthy(isoTimestamp: string): void {
+    healthy = true;
+    events.push({ type: 'HEALTHY', isoTimestamp });
+  }
+
+  async function tryRefreshEndpoint(page: Page, isoTimestamp: string): Promise<boolean> {
+    if (!config.refreshEndpoint) return false;
+
+    events.push({
+      type: 'REFRESH_ATTEMPT',
+      isoTimestamp,
+      strategy: 'refreshEndpoint',
+      attemptCount: refreshAttempts + 1,
+    });
+
+    try {
+      const result = await page.evaluate(async (endpoint) => {
+        try {
+          const res = await fetch(endpoint, { method: 'POST', credentials: 'include' });
+          if (res.ok) {
+            const data = await res.json().catch(() => ({}));
+            return { ok: true, token: (data as any).access_token || (data as any).token || null };
+          }
+          return { ok: false, token: null };
+        } catch {
+          return { ok: false, token: null };
+        }
+      }, config.refreshEndpoint);
+
+      if (result.ok) {
+        if (result.token && config.tokenLocalStorageKey) {
+          await page.evaluate(
+            ([key, val]) => localStorage.setItem(key, val),
+            [config.tokenLocalStorageKey, result.token] as [string, string],
+          );
+        }
+        refreshAttempts++;
+        events.push({ type: 'REFRESH_SUCCESS', isoTimestamp, strategy: 'refreshEndpoint' });
+        return true;
+      }
+    } catch {
+      // page.evaluate or localStorage.setItem failed — fall through
+    }
+
+    return false;
+  }
+
+  async function tryReLogin(page: Page, isoTimestamp: string): Promise<boolean> {
+    if (!config.loginUrl || !config.email || !config.password) return false;
+
+    events.push({
+      type: 'REFRESH_ATTEMPT',
+      isoTimestamp,
+      strategy: 'reLogin',
+      attemptCount: refreshAttempts + 1,
+    });
+
+    try {
+      await page.goto(config.loginUrl, { waitUntil: 'domcontentloaded' });
+
+      await page
+        .locator('input[type="email"], input[name="email"], input[name="username"]')
+        .first()
+        .fill(config.email);
+
+      await page
+        .locator('input[type="password"]')
+        .first()
+        .fill(config.password);
+
+      await page
+        .locator(
+          'button[type="submit"], button:has-text("Sign In"), button:has-text("Log In"), button:has-text("Login")',
+        )
+        .first()
+        .click();
+
+      await page.waitForFunction(
+        (loginUrl) => !location.href.includes(loginUrl),
+        config.loginUrl,
+        { timeout: 10000 },
+      );
+
+      events.push({ type: 'REFRESH_SUCCESS', isoTimestamp, strategy: 'reLogin' });
+      return true;
+    } catch {
+      events.push({ type: 'REFRESH_FAILED', isoTimestamp, strategy: 'reLogin' });
+      return false;
+    }
+  }
+
+  async function refresh(page: Page, isoTimestamp: string): Promise<boolean> {
+    // Strategy 1: token refresh endpoint
+    if (config.refreshEndpoint) {
+      const ok = await tryRefreshEndpoint(page, isoTimestamp);
+      if (ok) return true;
+    }
+
+    // Strategy 2: re-login
+    if (config.loginUrl && config.email && config.password) {
+      const ok = await tryReLogin(page, isoTimestamp);
+      if (ok) return true;
+    }
+
+    // All strategies exhausted
+    refreshAttempts++;
+    events.push({ type: 'REFRESH_FAILED', isoTimestamp, attemptCount: refreshAttempts });
+
+    if (refreshAttempts >= maxAttempts) {
+      authFailure = true;
+      events.push({ type: 'AUTH_FAILURE', isoTimestamp, attemptCount: refreshAttempts });
+    }
+
+    return false;
+  }
+
+  function getEvents(): SessionEvent[] {
+    return [...events];
+  }
+
+  function getStats(): { healthy: boolean; refreshAttempts: number; authFailure: boolean } {
+    return { healthy, refreshAttempts, authFailure };
+  }
+
+  return {
+    onResponse,
+    onNavigation,
+    isHealthy,
+    needsRefresh,
+    markHealthy,
+    refresh,
+    getEvents,
+    getStats,
+  };
+}
+
+/**
+ * Attaches a Playwright response listener to feed HTTP status codes into the
+ * session manager. Returns a cleanup function to remove the listener.
+ */
+export function attachSessionWatcher(page: Page, manager: SessionManager): () => void {
+  const onResponse = (res: import('playwright').Response) => {
+    manager.onResponse(res.url(), res.status());
+  };
+  page.on('response', onResponse);
+  return () => page.off('response', onResponse);
+}