@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/crawler.ts.rej

@@ -0,0 +1,12 @@
+@@ -2160,8 +2160,9 @@
+         
+         const [, elements, nextLinks, atElements] = await Promise.all([
+           page.screenshot({ path: shotPath, fullPage: true })
+-            .then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP')),
+-          extractElements(stagehand, page, elementLLM ?? undefined).catch(() => [] as Awaited<ReturnType<typeof extractElements>>),
++            .then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP'))
++            .catch((err) => console.error('[crawler] page.screenshot failed:', err)),
++          extractElements(stagehand, page, elementLLM ?? undefined).catch((err) => { console.error('[crawler] extractElements failed:', err); return [] as Awaited<ReturnType<typeof extractElements>> }),
+           extractNavigationLinks(page).catch(() => [] as string[]),
+           stagehand.page.extractElements({
+ 

package/src/crud-interaction.ts

@@ -0,0 +1,386 @@
+import type { Page, Locator } from 'playwright';
+
+// ── Types ─────────────────────────────────────────────────────────────────────
+
+export type EditFlowType = 'inline' | 'modal' | 'navigation' | 'unknown';
+
+export interface CrudFlowCapture {
+  entityName: string;
+  createTrigger?: string;
+  editFlowType: EditFlowType;
+  editFields: Array<{ name: string; type: string; prefilledValue?: string }>;
+  deleteConfirmation?: { message: string; confirmLabel: string; cancelLabel: string };
+  createSuccessToast?: string;
+  editSuccessToast?: string;
+  rowCount: number;
+  hasDeleteButton: boolean;
+  hasEditButton: boolean;
+}
+
+export interface CrudProbeResult {
+  flows: CrudFlowCapture[];
+  toastsObserved: Array<{ action: string; text: string; role: string }>;
+}
+
+// ── Internal: Toast observer ──────────────────────────────────────────────────
+
+function injectToastObserver(page: Page): void {
+  page.evaluate(() => {
+    if ((window as any).__toastObserver) return;
+
+    (window as any).__capturedToasts = [] as Array<{ text: string; role: string }>;
+
+    const TOAST_CLASSES = ['toast', 'snackbar', 'notification'];
+
+    function matchesToast(el: Element): boolean {
+      const role = el.getAttribute('role') ?? '';
+      if (role === 'alert' || role === 'status') return true;
+      const cls = (el.getAttribute('class') ?? '').toLowerCase();
+      return TOAST_CLASSES.some((t) => cls.includes(t));
+    }
+
+    function captureNode(node: Node): void {
+      if (!(node instanceof Element)) return;
+      if (matchesToast(node)) {
+        const text = ((node as HTMLElement).innerText ?? '').replace(/\s+/g, ' ').trim();
+        const role = node.getAttribute('role') ?? 'alert';
+        if (text) (window as any).__capturedToasts.push({ text, role });
+      }
+      node.querySelectorAll('[role="alert"],[role="status"],[class*="toast"],[class*="Toast"],[class*="snackbar"],[class*="notification"]').forEach((child) => {
+        const text = ((child as HTMLElement).innerText ?? '').replace(/\s+/g, ' ').trim();
+        const role = child.getAttribute('role') ?? 'alert';
+        if (text) (window as any).__capturedToasts.push({ text, role });
+      });
+    }
+
+    const observer = new MutationObserver((mutations) => {
+      for (const m of mutations) {
+        m.addedNodes.forEach(captureNode);
+      }
+    });
+
+    observer.observe(document.body, { childList: true, subtree: true });
+    (window as any).__toastObserver = observer;
+  }).catch(() => undefined);
+}
+
+async function collectCapturedToasts(page: Page): Promise<Array<{ text: string; role: string }>> {
+  try {
+    return await page.evaluate(() => {
+      const toasts: Array<{ text: string; role: string }> = (window as any).__capturedToasts ?? [];
+      (window as any).__capturedToasts = [];
+      return toasts;
+    });
+  } catch {
+    return [];
+  }
+}
+
+// ── Internal: Button discovery ────────────────────────────────────────────────
+
+interface CrudButtons {
+  editBtns: Locator[];
+  deleteBtns: Locator[];
+  createBtn: Locator | null;
+}
+
+async function findCrudButtons(page: Page): Promise<CrudButtons> {
+  const editLocator = page.locator(
+    'button[aria-label*="edit" i], button[aria-label*="Edit"], button:has-text("Edit")'
+  );
+  const deleteLocator = page.locator(
+    'button[aria-label*="delete" i], button[aria-label*="Delete"], button:has-text("Delete")'
+  );
+  const createLocator = page.locator(
+    'button:has-text("New"), button:has-text("Create"), button:has-text("Add")'
+  );
+
+  const [editCount, deleteCount, createCount] = await Promise.all([
+    editLocator.count().catch(() => 0),
+    deleteLocator.count().catch(() => 0),
+    createLocator.count().catch(() => 0),
+  ]);
+
+  const editBtns: Locator[] = [];
+  for (let i = 0; i < Math.min(editCount, 3); i++) {
+    editBtns.push(editLocator.nth(i));
+  }
+
+  const deleteBtns: Locator[] = [];
+  for (let i = 0; i < Math.min(deleteCount, 3); i++) {
+    deleteBtns.push(deleteLocator.nth(i));
+  }
+
+  const createBtn = createCount > 0 ? createLocator.first() : null;
+
+  return { editBtns, deleteBtns, createBtn };
+}
+
+// ── Internal: Edit flow detection ─────────────────────────────────────────────
+
+async function detectEditFlowType(page: Page, editButton: Locator): Promise<EditFlowType> {
+  try {
+    const urlBefore = page.url();
+    await editButton.click({ timeout: 2000 });
+    await page.waitForTimeout(600);
+
+    const urlAfter = page.url();
+    if (urlAfter !== urlBefore) {
+      await page.goBack().catch(() => undefined);
+      await page.waitForTimeout(300);
+      return 'navigation';
+    }
+
+    const dialogVisible = await page.locator('[role="dialog"]').first().isVisible().catch(() => false);
+    if (dialogVisible) {
+      await page.keyboard.press('Escape');
+      await page.waitForTimeout(300);
+      return 'modal';
+    }
+
+    const inlineInputCount = await page.locator('input:visible, textarea:visible').count().catch(() => 0);
+    if (inlineInputCount > 0) {
+      await page.keyboard.press('Escape');
+      await page.waitForTimeout(300);
+      return 'inline';
+    }
+
+    await page.keyboard.press('Escape');
+    await page.waitForTimeout(300);
+    return 'unknown';
+  } catch {
+    try {
+      await page.keyboard.press('Escape');
+    } catch { /* ignore */ }
+    return 'unknown';
+  }
+}
+
+// ── Internal: Modal field capture ─────────────────────────────────────────────
+
+async function captureEditModal(
+  page: Page
+): Promise<Array<{ name: string; type: string; prefilledValue?: string }>> {
+  try {
+    const dialog = page.locator('[role="dialog"]').first();
+    const fields = dialog.locator('input, textarea, select');
+    const count = Math.min(await fields.count().catch(() => 0), 20);
+
+    const results: Array<{ name: string; type: string; prefilledValue?: string }> = [];
+
+    for (let i = 0; i < count; i++) {
+      try {
+        const field = fields.nth(i);
+        const tagName = await field.evaluate((el) => el.tagName.toLowerCase()).catch(() => 'input');
+        const inputType = await field.getAttribute('type').catch(() => null) ?? tagName;
+        const name =
+          (await field.getAttribute('name').catch(() => null)) ??
+          (await field.getAttribute('id').catch(() => null)) ??
+          (await field.getAttribute('aria-label').catch(() => null)) ??
+          `field_${i}`;
+        const prefilledValue = await field.inputValue().catch(() => undefined);
+
+        results.push({
+          name,
+          type: inputType,
+          prefilledValue: prefilledValue || undefined,
+        });
+      } catch { /* skip individual field errors */ }
+    }
+
+    return results;
+  } catch {
+    return [];
+  }
+}
+
+// ── Internal: Delete confirmation capture ─────────────────────────────────────
+
+async function captureDeleteConfirmation(
+  page: Page,
+  deleteBtn: Locator
+): Promise<{ message: string; confirmLabel: string; cancelLabel: string } | null> {
+  try {
+    await deleteBtn.click({ timeout: 2000 });
+
+    const confirmDialog = page.locator('[role="alertdialog"], [role="dialog"]').first();
+    try {
+      await confirmDialog.waitFor({ state: 'visible', timeout: 2000 });
+    } catch {
+      await page.keyboard.press('Escape');
+      return null;
+    }
+
+    const dialogText = await confirmDialog.evaluate((el) =>
+      (el as HTMLElement).innerText?.replace(/\s+/g, ' ').trim()
+    ).catch(() => '');
+
+    const isDeleteDialog =
+      /delete|remove|are you sure|confirm/i.test(dialogText);
+
+    if (!isDeleteDialog) {
+      await page.keyboard.press('Escape');
+      return null;
+    }
+
+    // Capture button labels inside the dialog
+    const buttons = confirmDialog.locator('button');
+    const btnCount = await buttons.count().catch(() => 0);
+    const btnLabels: string[] = [];
+    for (let i = 0; i < btnCount; i++) {
+      try {
+        const label = await buttons.nth(i).innerText();
+        btnLabels.push(label.trim());
+      } catch { /* skip */ }
+    }
+
+    // Heuristic: confirm = first destructive-looking label, cancel = cancel/no label
+    const confirmLabel =
+      btnLabels.find((l) => /delete|remove|yes|confirm/i.test(l)) ??
+      btnLabels[0] ??
+      'Confirm';
+    const cancelLabel =
+      btnLabels.find((l) => /cancel|no|keep|dismiss/i.test(l)) ??
+      btnLabels[btnLabels.length - 1] ??
+      'Cancel';
+
+    // CRITICAL: Always cancel — never actually delete data
+    await page.keyboard.press('Escape');
+    await page.waitForTimeout(300);
+
+    return {
+      message: dialogText,
+      confirmLabel,
+      cancelLabel,
+    };
+  } catch {
+    try {
+      await page.keyboard.press('Escape');
+    } catch { /* ignore */ }
+    return null;
+  }
+}
+
+// ── Internal: Row count ───────────────────────────────────────────────────────
+
+async function detectRowCount(page: Page): Promise<number> {
+  try {
+    // Try common table row patterns
+    const tableRows = await page.locator('table tbody tr').count().catch(() => 0);
+    if (tableRows > 0) return tableRows;
+
+    const listItems = await page.locator('[role="row"]').count().catch(() => 0);
+    if (listItems > 0) return listItems;
+
+    const dataItems = await page
+      .locator('[data-testid*="row"], [class*="row" i], [class*="item" i]')
+      .count()
+      .catch(() => 0);
+    return dataItems;
+  } catch {
+    return 0;
+  }
+}
+
+// ── Internal: Create trigger label ────────────────────────────────────────────
+
+async function getCreateTriggerLabel(createBtn: Locator | null): Promise<string | undefined> {
+  if (!createBtn) return undefined;
+  try {
+    const label = await createBtn.innerText();
+    return label.trim() || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+// ── Public: probeCrudFlows ────────────────────────────────────────────────────
+
+export async function probeCrudFlows(
+  page: Page,
+  entityName = 'Entity'
+): Promise<CrudProbeResult> {
+  const toastsObserved: Array<{ action: string; text: string; role: string }> = [];
+
+  // Step 1: Inject toast observer
+  injectToastObserver(page);
+
+  // Step 2: Find CRUD buttons
+  const { editBtns, deleteBtns, createBtn } = await findCrudButtons(page);
+
+  // Step 3: Row count
+  const rowCount = await detectRowCount(page);
+
+  // Step 4: Create trigger label
+  const createTrigger = await getCreateTriggerLabel(createBtn);
+
+  // Step 5: Probe edit flow (use first edit button)
+  let editFlowType: EditFlowType = 'unknown';
+  let editFields: Array<{ name: string; type: string; prefilledValue?: string }> = [];
+
+  if (editBtns.length > 0) {
+    try {
+      editFlowType = await detectEditFlowType(page, editBtns[0]);
+
+      if (editFlowType === 'modal') {
+        // Re-open to capture fields
+        try {
+          await editBtns[0].click({ timeout: 2000 });
+          await page.waitForTimeout(400);
+          const dialogVisible = await page
+            .locator('[role="dialog"]')
+            .first()
+            .isVisible()
+            .catch(() => false);
+          if (dialogVisible) {
+            editFields = await captureEditModal(page);
+            const editToasts = await collectCapturedToasts(page);
+            for (const t of editToasts) {
+              toastsObserved.push({ action: 'edit_open', ...t });
+            }
+          }
+          await page.keyboard.press('Escape');
+          await page.waitForTimeout(300);
+        } catch { /* non-fatal */ }
+      }
+    } catch { /* non-fatal */ }
+  }
+
+  // Step 6: Probe delete confirmation (NEVER actually delete)
+  let deleteConfirmation: CrudFlowCapture['deleteConfirmation'];
+
+  if (deleteBtns.length > 0) {
+    try {
+      const result = await captureDeleteConfirmation(page, deleteBtns[0]);
+      if (result) {
+        deleteConfirmation = result;
+        const deleteToasts = await collectCapturedToasts(page);
+        for (const t of deleteToasts) {
+          toastsObserved.push({ action: 'delete_cancel', ...t });
+        }
+      }
+    } catch { /* non-fatal */ }
+  }
+
+  // Step 7: Collect any remaining toasts
+  const remainingToasts = await collectCapturedToasts(page);
+  for (const t of remainingToasts) {
+    toastsObserved.push({ action: 'ambient', ...t });
+  }
+
+  const flow: CrudFlowCapture = {
+    entityName,
+    createTrigger,
+    editFlowType,
+    editFields,
+    deleteConfirmation,
+    rowCount,
+    hasDeleteButton: deleteBtns.length > 0,
+    hasEditButton: editBtns.length > 0,
+  };
+
+  return {
+    flows: [flow],
+    toastsObserved,
+  };
+}

package/src/custom-js-injector.ts

@@ -0,0 +1,93 @@
+/**
+ * Custom JS injector — run user-provided JavaScript in the context of each crawled page.
+ *
+ * Use cases:
+ * - Extract custom data points not captured by standard extractors
+ * - Set localStorage/sessionStorage values before crawl
+ * - Override fetch() to intercept API calls
+ * - Simulate user events (keydown, focus, etc.)
+ * - Bypass specific anti-bot checks
+ *
+ * Security: scripts run in the page context (sandboxed by Chromium's renderer).
+ * They cannot access the Node.js process or file system.
+ * ZeTa validates scripts are strings before injection.
+ */
+
+import type { Page } from 'playwright';
+
+export interface JsInjectionOpts {
+  /** Script to run on every page BEFORE navigation (via page.addInitScript) */
+  initScript?: string;
+  /** Script to run on every page AFTER navigation completes (via page.evaluate) */
+  postLoadScript?: string;
+  /** Key-value pairs to set in localStorage before page load */
+  localStorage?: Record<string, string>;
+  /** Key-value pairs to set in sessionStorage before page load */
+  sessionStorage?: Record<string, string>;
+  /** Cookie key-value pairs to add before page load */
+  cookies?: Array<{ name: string; value: string; domain?: string; path?: string }>;
+}
+
+export interface JsInjectionResult {
+  initScriptApplied: boolean;
+  postLoadResult: unknown;
+  storageSet: boolean;
+  cookiesSet: number;
+  error?: string;
+}
+
+export async function injectCustomJs(page: Page, opts: JsInjectionOpts): Promise<JsInjectionResult> {
+  const result: JsInjectionResult = {
+    initScriptApplied: false,
+    postLoadResult: null,
+    storageSet: false,
+    cookiesSet: 0,
+  };
+
+  // Set cookies via Playwright context
+  if (opts.cookies?.length) {
+    try {
+      await page.context().addCookies(
+        opts.cookies.map(c => ({ name: c.name, value: c.value, domain: c.domain ?? '', path: c.path ?? '/', url: page.url() || undefined }))
+      );
+      result.cookiesSet = opts.cookies.length;
+    } catch (e: any) {
+      result.error = `Cookie injection failed: ${e.message}`;
+    }
+  }
+
+  // Init script (runs before page load on every navigation)
+  if (opts.initScript || opts.localStorage || opts.sessionStorage) {
+    const storageSetup = [
+      opts.localStorage ? `Object.entries(${JSON.stringify(opts.localStorage)}).forEach(([k,v])=>localStorage.setItem(k,v));` : '',
+      opts.sessionStorage ? `Object.entries(${JSON.stringify(opts.sessionStorage)}).forEach(([k,v])=>sessionStorage.setItem(k,v));` : '',
+    ].filter(Boolean).join('\n');
+
+    const combinedInit = [storageSetup, opts.initScript ?? ''].filter(Boolean).join('\n');
+    try {
+      await page.addInitScript(combinedInit);
+      result.initScriptApplied = true;
+      result.storageSet = !!(opts.localStorage || opts.sessionStorage);
+    } catch (e: any) {
+      result.error = `Init script failed: ${e.message}`;
+    }
+  }
+
+  // Post-load script (runs after navigation)
+  if (opts.postLoadScript) {
+    try {
+      result.postLoadResult = await page.evaluate(opts.postLoadScript);
+    } catch (e: any) {
+      result.error = `Post-load script failed: ${e.message}`;
+    }
+  }
+
+  return result;
+}
+
+/** Safe string validator — ensures script is a string, not an object/function reference */
+export function validateScript(script: unknown): string {
+  if (typeof script !== 'string') throw new Error('Script must be a string');
+  if (script.length > 50_000) throw new Error('Script too large (>50KB)');
+  return script;
+}

package/src/dark-mode-capture.ts

@@ -0,0 +1,110 @@
+/**
+ * Dark mode capture — screenshot pages in both light and dark color schemes.
+ *
+ * Uses Playwright's page.emulateMedia({ colorScheme: 'dark' }) to force dark mode,
+ * then captures a screenshot. Compares with light mode to determine if dark mode
+ * is supported (significant visual difference).
+ *
+ * Also detects prefers-color-scheme media query support in CSS.
+ */
+
+import type { Page } from 'playwright';
+
+export interface DarkModeCapture {
+  url: string;
+  lightScreenshot?: Buffer;
+  darkScreenshot?: Buffer;
+  hasDarkMode: boolean;
+  hasCssDarkMode: boolean; // @media (prefers-color-scheme: dark) detected in stylesheets
+  backgroundColorLight?: string;
+  backgroundColorDark?: string;
+  capturedAt: string;
+}
+
+async function getBackgroundColor(page: Page): Promise<string> {
+  try {
+    return await page.evaluate(() => {
+      const body = document.body;
+      return window.getComputedStyle(body).backgroundColor;
+    });
+  } catch {
+    return 'unknown';
+  }
+}
+
+function colorsAreDifferent(a: string, b: string): boolean {
+  if (a === b) return false;
+  if (a === 'unknown' || b === 'unknown') return false;
+  // Parse rgb(r, g, b) and compare luminance
+  const parse = (c: string) => {
+    const m = c.match(/\d+/g);
+    return m ? m.map(Number) : [0, 0, 0];
+  };
+  const [ar, ag, ab] = parse(a);
+  const [br, bg, bb] = parse(b);
+  const lumA = 0.299 * ar + 0.587 * ag + 0.114 * ab;
+  const lumB = 0.299 * br + 0.587 * bg + 0.114 * bb;
+  return Math.abs(lumA - lumB) > 30; // significant luminance difference
+}
+
+async function hasCssDarkModeSupport(page: Page): Promise<boolean> {
+  try {
+    return await page.evaluate(() => {
+      const sheets = Array.from(document.styleSheets);
+      for (const sheet of sheets) {
+        try {
+          const rules = Array.from(sheet.cssRules ?? []);
+          for (const rule of rules) {
+            if (rule instanceof CSSMediaRule && rule.conditionText?.includes('prefers-color-scheme')) {
+              return true;
+            }
+          }
+        } catch { /* cross-origin stylesheet — skip */ }
+      }
+      return false;
+    });
+  } catch {
+    return false;
+  }
+}
+
+export async function captureDarkMode(
+  page: Page,
+  url: string,
+  opts: { takeScreenshots?: boolean } = {},
+): Promise<DarkModeCapture> {
+  const capturedAt = new Date().toISOString();
+  const { takeScreenshots = false } = opts;
+
+  // Light mode
+  await page.emulateMedia({ colorScheme: 'light' }).catch(() => {});
+  const backgroundColorLight = await getBackgroundColor(page);
+  const hasCssDark = await hasCssDarkModeSupport(page);
+  const lightScreenshot = takeScreenshots
+    ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
+    : undefined;
+
+  // Dark mode
+  await page.emulateMedia({ colorScheme: 'dark' }).catch(() => {});
+  await page.waitForTimeout(300); // allow CSS transitions
+  const backgroundColorDark = await getBackgroundColor(page);
+  const darkScreenshot = takeScreenshots
+    ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
+    : undefined;
+
+  // Restore light mode
+  await page.emulateMedia({ colorScheme: 'light' }).catch(() => {});
+
+  const hasDarkMode = colorsAreDifferent(backgroundColorLight, backgroundColorDark) || hasCssDark;
+
+  return {
+    url,
+    lightScreenshot,
+    darkScreenshot,
+    hasDarkMode,
+    hasCssDarkMode: hasCssDark,
+    backgroundColorLight,
+    backgroundColorDark,
+    capturedAt,
+  };
+}

package/src/dead-link-detector.ts

@@ -0,0 +1,76 @@
+/**
+ * Dead link detector — tracks broken links discovered during crawl.
+ *
+ * A "dead link" is a URL that:
+ * - Returns 404, 410, or other 4xx/5xx responses
+ * - Times out after maxRetries
+ * - Redirects to an error page (detected by title heuristic)
+ *
+ * Results: list of broken links with source page, status code, and anchor text.
+ * Used for: SEO audit export, test report enrichment.
+ */
+
+export interface DeadLink {
+  url: string;
+  statusCode: number;
+  sourceUrl?: string;
+  anchorText?: string;
+  checkedAt: string;
+  reason: 'http-error' | 'timeout' | 'dns-error';
+}
+
+export class DeadLinkTracker {
+  private deadLinks: DeadLink[] = [];
+  private checked = new Map<string, number>(); // url → statusCode
+
+  record(
+    url: string,
+    statusCode: number,
+    source?: { sourceUrl?: string; anchorText?: string },
+  ): void {
+    if (this.checked.has(url)) return;
+    this.checked.set(url, statusCode);
+    if (statusCode >= 400) {
+      this.deadLinks.push({
+        url,
+        statusCode,
+        sourceUrl: source?.sourceUrl,
+        anchorText: source?.anchorText,
+        checkedAt: new Date().toISOString(),
+        reason: statusCode === 0 ? 'timeout' : 'http-error',
+      });
+    }
+  }
+
+  recordTimeout(url: string, sourceUrl?: string): void {
+    if (this.checked.has(url)) return;
+    this.checked.set(url, 0);
+    this.deadLinks.push({ url, statusCode: 0, sourceUrl, checkedAt: new Date().toISOString(), reason: 'timeout' });
+  }
+
+  getDeadLinks(): DeadLink[] {
+    return [...this.deadLinks];
+  }
+
+  isKnownBroken(url: string): boolean {
+    const code = this.checked.get(url);
+    return code !== undefined && code >= 400;
+  }
+
+  stats(): { total: number; by404: number; by5xx: number; byTimeout: number } {
+    return {
+      total: this.deadLinks.length,
+      by404: this.deadLinks.filter(l => l.statusCode === 404).length,
+      by5xx: this.deadLinks.filter(l => l.statusCode >= 500).length,
+      byTimeout: this.deadLinks.filter(l => l.reason === 'timeout').length,
+    };
+  }
+
+  toCsv(): string {
+    const rows = [['URL', 'Status', 'Source URL', 'Anchor Text', 'Checked At']];
+    for (const l of this.deadLinks) {
+      rows.push([l.url, String(l.statusCode), l.sourceUrl ?? '', l.anchorText ?? '', l.checkedAt]);
+    }
+    return rows.map(r => r.map(c => `"${c.replace(/"/g, '""')}"`).join(',')).join('\n');
+  }
+}