@detiq/crawler 1.1.13 → 1.1.15

package/src/crud-interaction.ts

@@ -0,0 +1,386 @@
+import type { Page, Locator } from 'playwright';
+
+// ── Types ─────────────────────────────────────────────────────────────────────
+
+export type EditFlowType = 'inline' | 'modal' | 'navigation' | 'unknown';
+
+export interface CrudFlowCapture {
+  entityName: string;
+  createTrigger?: string;
+  editFlowType: EditFlowType;
+  editFields: Array<{ name: string; type: string; prefilledValue?: string }>;
+  deleteConfirmation?: { message: string; confirmLabel: string; cancelLabel: string };
+  createSuccessToast?: string;
+  editSuccessToast?: string;
+  rowCount: number;
+  hasDeleteButton: boolean;
+  hasEditButton: boolean;
+}
+
+export interface CrudProbeResult {
+  flows: CrudFlowCapture[];
+  toastsObserved: Array<{ action: string; text: string; role: string }>;
+}
+
+// ── Internal: Toast observer ──────────────────────────────────────────────────
+
+function injectToastObserver(page: Page): void {
+  page.evaluate(() => {
+    if ((window as any).__toastObserver) return;
+
+    (window as any).__capturedToasts = [] as Array<{ text: string; role: string }>;
+
+    const TOAST_CLASSES = ['toast', 'snackbar', 'notification'];
+
+    function matchesToast(el: Element): boolean {
+      const role = el.getAttribute('role') ?? '';
+      if (role === 'alert' || role === 'status') return true;
+      const cls = (el.getAttribute('class') ?? '').toLowerCase();
+      return TOAST_CLASSES.some((t) => cls.includes(t));
+    }
+
+    function captureNode(node: Node): void {
+      if (!(node instanceof Element)) return;
+      if (matchesToast(node)) {
+        const text = ((node as HTMLElement).innerText ?? '').replace(/\s+/g, ' ').trim();
+        const role = node.getAttribute('role') ?? 'alert';
+        if (text) (window as any).__capturedToasts.push({ text, role });
+      }
+      node.querySelectorAll('[role="alert"],[role="status"],[class*="toast"],[class*="Toast"],[class*="snackbar"],[class*="notification"]').forEach((child) => {
+        const text = ((child as HTMLElement).innerText ?? '').replace(/\s+/g, ' ').trim();
+        const role = child.getAttribute('role') ?? 'alert';
+        if (text) (window as any).__capturedToasts.push({ text, role });
+      });
+    }
+
+    const observer = new MutationObserver((mutations) => {
+      for (const m of mutations) {
+        m.addedNodes.forEach(captureNode);
+      }
+    });
+
+    observer.observe(document.body, { childList: true, subtree: true });
+    (window as any).__toastObserver = observer;
+  }).catch(() => undefined);
+}
+
+async function collectCapturedToasts(page: Page): Promise<Array<{ text: string; role: string }>> {
+  try {
+    return await page.evaluate(() => {
+      const toasts: Array<{ text: string; role: string }> = (window as any).__capturedToasts ?? [];
+      (window as any).__capturedToasts = [];
+      return toasts;
+    });
+  } catch {
+    return [];
+  }
+}
+
+// ── Internal: Button discovery ────────────────────────────────────────────────
+
+interface CrudButtons {
+  editBtns: Locator[];
+  deleteBtns: Locator[];
+  createBtn: Locator | null;
+}
+
+async function findCrudButtons(page: Page): Promise<CrudButtons> {
+  const editLocator = page.locator(
+    'button[aria-label*="edit" i], button[aria-label*="Edit"], button:has-text("Edit")'
+  );
+  const deleteLocator = page.locator(
+    'button[aria-label*="delete" i], button[aria-label*="Delete"], button:has-text("Delete")'
+  );
+  const createLocator = page.locator(
+    'button:has-text("New"), button:has-text("Create"), button:has-text("Add")'
+  );
+
+  const [editCount, deleteCount, createCount] = await Promise.all([
+    editLocator.count().catch(() => 0),
+    deleteLocator.count().catch(() => 0),
+    createLocator.count().catch(() => 0),
+  ]);
+
+  const editBtns: Locator[] = [];
+  for (let i = 0; i < Math.min(editCount, 3); i++) {
+    editBtns.push(editLocator.nth(i));
+  }
+
+  const deleteBtns: Locator[] = [];
+  for (let i = 0; i < Math.min(deleteCount, 3); i++) {
+    deleteBtns.push(deleteLocator.nth(i));
+  }
+
+  const createBtn = createCount > 0 ? createLocator.first() : null;
+
+  return { editBtns, deleteBtns, createBtn };
+}
+
+// ── Internal: Edit flow detection ─────────────────────────────────────────────
+
+async function detectEditFlowType(page: Page, editButton: Locator): Promise<EditFlowType> {
+  try {
+    const urlBefore = page.url();
+    await editButton.click({ timeout: 2000 });
+    await page.waitForTimeout(600);
+
+    const urlAfter = page.url();
+    if (urlAfter !== urlBefore) {
+      await page.goBack().catch(() => undefined);
+      await page.waitForTimeout(300);
+      return 'navigation';
+    }
+
+    const dialogVisible = await page.locator('[role="dialog"]').first().isVisible().catch(() => false);
+    if (dialogVisible) {
+      await page.keyboard.press('Escape');
+      await page.waitForTimeout(300);
+      return 'modal';
+    }
+
+    const inlineInputCount = await page.locator('input:visible, textarea:visible').count().catch(() => 0);
+    if (inlineInputCount > 0) {
+      await page.keyboard.press('Escape');
+      await page.waitForTimeout(300);
+      return 'inline';
+    }
+
+    await page.keyboard.press('Escape');
+    await page.waitForTimeout(300);
+    return 'unknown';
+  } catch {
+    try {
+      await page.keyboard.press('Escape');
+    } catch { /* ignore */ }
+    return 'unknown';
+  }
+}
+
+// ── Internal: Modal field capture ─────────────────────────────────────────────
+
+async function captureEditModal(
+  page: Page
+): Promise<Array<{ name: string; type: string; prefilledValue?: string }>> {
+  try {
+    const dialog = page.locator('[role="dialog"]').first();
+    const fields = dialog.locator('input, textarea, select');
+    const count = Math.min(await fields.count().catch(() => 0), 20);
+
+    const results: Array<{ name: string; type: string; prefilledValue?: string }> = [];
+
+    for (let i = 0; i < count; i++) {
+      try {
+        const field = fields.nth(i);
+        const tagName = await field.evaluate((el) => el.tagName.toLowerCase()).catch(() => 'input');
+        const inputType = await field.getAttribute('type').catch(() => null) ?? tagName;
+        const name =
+          (await field.getAttribute('name').catch(() => null)) ??
+          (await field.getAttribute('id').catch(() => null)) ??
+          (await field.getAttribute('aria-label').catch(() => null)) ??
+          `field_${i}`;
+        const prefilledValue = await field.inputValue().catch(() => undefined);
+
+        results.push({
+          name,
+          type: inputType,
+          prefilledValue: prefilledValue || undefined,
+        });
+      } catch { /* skip individual field errors */ }
+    }
+
+    return results;
+  } catch {
+    return [];
+  }
+}
+
+// ── Internal: Delete confirmation capture ─────────────────────────────────────
+
+async function captureDeleteConfirmation(
+  page: Page,
+  deleteBtn: Locator
+): Promise<{ message: string; confirmLabel: string; cancelLabel: string } | null> {
+  try {
+    await deleteBtn.click({ timeout: 2000 });
+
+    const confirmDialog = page.locator('[role="alertdialog"], [role="dialog"]').first();
+    try {
+      await confirmDialog.waitFor({ state: 'visible', timeout: 2000 });
+    } catch {
+      await page.keyboard.press('Escape');
+      return null;
+    }
+
+    const dialogText = await confirmDialog.evaluate((el) =>
+      (el as HTMLElement).innerText?.replace(/\s+/g, ' ').trim()
+    ).catch(() => '');
+
+    const isDeleteDialog =
+      /delete|remove|are you sure|confirm/i.test(dialogText);
+
+    if (!isDeleteDialog) {
+      await page.keyboard.press('Escape');
+      return null;
+    }
+
+    // Capture button labels inside the dialog
+    const buttons = confirmDialog.locator('button');
+    const btnCount = await buttons.count().catch(() => 0);
+    const btnLabels: string[] = [];
+    for (let i = 0; i < btnCount; i++) {
+      try {
+        const label = await buttons.nth(i).innerText();
+        btnLabels.push(label.trim());
+      } catch { /* skip */ }
+    }
+
+    // Heuristic: confirm = first destructive-looking label, cancel = cancel/no label
+    const confirmLabel =
+      btnLabels.find((l) => /delete|remove|yes|confirm/i.test(l)) ??
+      btnLabels[0] ??
+      'Confirm';
+    const cancelLabel =
+      btnLabels.find((l) => /cancel|no|keep|dismiss/i.test(l)) ??
+      btnLabels[btnLabels.length - 1] ??
+      'Cancel';
+
+    // CRITICAL: Always cancel — never actually delete data
+    await page.keyboard.press('Escape');
+    await page.waitForTimeout(300);
+
+    return {
+      message: dialogText,
+      confirmLabel,
+      cancelLabel,
+    };
+  } catch {
+    try {
+      await page.keyboard.press('Escape');
+    } catch { /* ignore */ }
+    return null;
+  }
+}
+
+// ── Internal: Row count ───────────────────────────────────────────────────────
+
+async function detectRowCount(page: Page): Promise<number> {
+  try {
+    // Try common table row patterns
+    const tableRows = await page.locator('table tbody tr').count().catch(() => 0);
+    if (tableRows > 0) return tableRows;
+
+    const listItems = await page.locator('[role="row"]').count().catch(() => 0);
+    if (listItems > 0) return listItems;
+
+    const dataItems = await page
+      .locator('[data-testid*="row"], [class*="row" i], [class*="item" i]')
+      .count()
+      .catch(() => 0);
+    return dataItems;
+  } catch {
+    return 0;
+  }
+}
+
+// ── Internal: Create trigger label ────────────────────────────────────────────
+
+async function getCreateTriggerLabel(createBtn: Locator | null): Promise<string | undefined> {
+  if (!createBtn) return undefined;
+  try {
+    const label = await createBtn.innerText();
+    return label.trim() || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+// ── Public: probeCrudFlows ────────────────────────────────────────────────────
+
+export async function probeCrudFlows(
+  page: Page,
+  entityName = 'Entity'
+): Promise<CrudProbeResult> {
+  const toastsObserved: Array<{ action: string; text: string; role: string }> = [];
+
+  // Step 1: Inject toast observer
+  injectToastObserver(page);
+
+  // Step 2: Find CRUD buttons
+  const { editBtns, deleteBtns, createBtn } = await findCrudButtons(page);
+
+  // Step 3: Row count
+  const rowCount = await detectRowCount(page);
+
+  // Step 4: Create trigger label
+  const createTrigger = await getCreateTriggerLabel(createBtn);
+
+  // Step 5: Probe edit flow (use first edit button)
+  let editFlowType: EditFlowType = 'unknown';
+  let editFields: Array<{ name: string; type: string; prefilledValue?: string }> = [];
+
+  if (editBtns.length > 0) {
+    try {
+      editFlowType = await detectEditFlowType(page, editBtns[0]);
+
+      if (editFlowType === 'modal') {
+        // Re-open to capture fields
+        try {
+          await editBtns[0].click({ timeout: 2000 });
+          await page.waitForTimeout(400);
+          const dialogVisible = await page
+            .locator('[role="dialog"]')
+            .first()
+            .isVisible()
+            .catch(() => false);
+          if (dialogVisible) {
+            editFields = await captureEditModal(page);
+            const editToasts = await collectCapturedToasts(page);
+            for (const t of editToasts) {
+              toastsObserved.push({ action: 'edit_open', ...t });
+            }
+          }
+          await page.keyboard.press('Escape');
+          await page.waitForTimeout(300);
+        } catch { /* non-fatal */ }
+      }
+    } catch { /* non-fatal */ }
+  }
+
+  // Step 6: Probe delete confirmation (NEVER actually delete)
+  let deleteConfirmation: CrudFlowCapture['deleteConfirmation'];
+
+  if (deleteBtns.length > 0) {
+    try {
+      const result = await captureDeleteConfirmation(page, deleteBtns[0]);
+      if (result) {
+        deleteConfirmation = result;
+        const deleteToasts = await collectCapturedToasts(page);
+        for (const t of deleteToasts) {
+          toastsObserved.push({ action: 'delete_cancel', ...t });
+        }
+      }
+    } catch { /* non-fatal */ }
+  }
+
+  // Step 7: Collect any remaining toasts
+  const remainingToasts = await collectCapturedToasts(page);
+  for (const t of remainingToasts) {
+    toastsObserved.push({ action: 'ambient', ...t });
+  }
+
+  const flow: CrudFlowCapture = {
+    entityName,
+    createTrigger,
+    editFlowType,
+    editFields,
+    deleteConfirmation,
+    rowCount,
+    hasDeleteButton: deleteBtns.length > 0,
+    hasEditButton: editBtns.length > 0,
+  };
+
+  return {
+    flows: [flow],
+    toastsObserved,
+  };
+}

package/src/dropdown-enumerator.ts

@@ -0,0 +1,312 @@
+import type { Page } from 'playwright';
+
+export interface DropdownOption {
+  value: string;
+  label: string;
+  disabled?: boolean;
+}
+
+export interface DropdownDef {
+  kind: 'native-select' | 'combobox' | 'listbox' | 'multiselect';
+  label: string;
+  locatorHint: string;
+  options: DropdownOption[];
+  defaultValue?: string;
+  cascadesFrom?: string;
+}
+
+export interface DropdownEnumerationResult {
+  dropdowns: DropdownDef[];
+  conditionalRevealMap: Record<string, string[]>;
+  totalOptionsFound: number;
+}
+
+async function enumerateNativeSelects(page: Page): Promise<DropdownDef[]> {
+  const selects = await page.$$('select:not([multiple])');
+  const limited = selects.slice(0, 50);
+  const results: DropdownDef[] = [];
+
+  for (const handle of limited) {
+    try {
+      const data = await handle.evaluate((el: HTMLSelectElement) => {
+        const name = el.name || el.id || '';
+        const ariaLabel = el.getAttribute('aria-label') || '';
+        let labelText = ariaLabel;
+        if (!labelText && el.id) {
+          const lbl = document.querySelector(`label[for="${el.id}"]`);
+          if (lbl) labelText = lbl.textContent?.trim() || '';
+        }
+        if (!labelText && name) labelText = name;
+        const options = Array.from(el.options).map((o) => ({
+          value: o.value,
+          label: o.text.trim(),
+          disabled: o.disabled || undefined,
+        }));
+        const selectedOption = el.options[el.selectedIndex];
+        const defaultValue = selectedOption ? selectedOption.value : undefined;
+        const locatorHint = name
+          ? `select[name="${name}"]`
+          : el.id
+          ? `select#${el.id}`
+          : 'select';
+        return { label: labelText, locatorHint, options, defaultValue };
+      });
+
+      results.push({
+        kind: 'native-select',
+        label: data.label,
+        locatorHint: data.locatorHint,
+        options: data.options,
+        defaultValue: data.defaultValue,
+      });
+    } catch {
+      // skip unresponsive handle
+    }
+  }
+
+  return results;
+}
+
+async function enumerateCustomDropdowns(page: Page): Promise<DropdownDef[]> {
+  const selectors = [
+    '[role="combobox"]',
+    '[aria-haspopup="listbox"]:not([role="combobox"])',
+    '[data-radix-select-trigger]',
+  ];
+
+  const seen = new Set<string>();
+  const handles: { handle: Awaited<ReturnType<Page['$$']>>[0]; hint: string }[] = [];
+
+  for (const sel of selectors) {
+    const els = await page.$$(sel);
+    for (const el of els) {
+      const hint = await el
+        .evaluate((e: Element) => {
+          const id = e.id ? `#${e.id}` : '';
+          const aria = e.getAttribute('aria-label') || '';
+          return id || aria || e.tagName.toLowerCase();
+        })
+        .catch(() => '');
+      if (!seen.has(hint)) {
+        seen.add(hint);
+        handles.push({ handle: el, hint });
+      }
+    }
+  }
+
+  const limited = handles.slice(0, 20);
+  const results: DropdownDef[] = [];
+
+  for (const { handle, hint } of limited) {
+    try {
+      const ariaLabel = await handle
+        .evaluate((e: Element) => e.getAttribute('aria-label') || e.textContent?.trim() || '')
+        .catch(() => '');
+
+      await handle.click();
+      await page.waitForSelector('[role="listbox"]', { timeout: 2000 });
+
+      const options = await page.$$eval('[role="listbox"] [role="option"]', (els) =>
+        els.map((el) => ({
+          value: el.getAttribute('data-value') || el.getAttribute('value') || el.textContent?.trim() || '',
+          label: el.textContent?.trim() || '',
+          disabled: el.getAttribute('aria-disabled') === 'true' || undefined,
+        }))
+      );
+
+      await page.keyboard.press('Escape');
+      await page.waitForTimeout(300);
+
+      results.push({
+        kind: 'combobox',
+        label: ariaLabel,
+        locatorHint: hint,
+        options,
+      });
+    } catch {
+      try {
+        await page.keyboard.press('Escape');
+        await page.waitForTimeout(300);
+      } catch {
+        // ignore cleanup failure
+      }
+    }
+  }
+
+  return results;
+}
+
+async function enumerateMultiSelects(page: Page): Promise<DropdownDef[]> {
+  const results: DropdownDef[] = [];
+
+  const nativeMulti = await page.$$('select[multiple]');
+  for (const handle of nativeMulti) {
+    try {
+      const data = await handle.evaluate((el: HTMLSelectElement) => {
+        const name = el.name || el.id || '';
+        const ariaLabel = el.getAttribute('aria-label') || '';
+        let labelText = ariaLabel;
+        if (!labelText && el.id) {
+          const lbl = document.querySelector(`label[for="${el.id}"]`);
+          if (lbl) labelText = lbl.textContent?.trim() || '';
+        }
+        if (!labelText && name) labelText = name;
+        const options = Array.from(el.options).map((o) => ({
+          value: o.value,
+          label: o.text.trim(),
+          disabled: o.disabled || undefined,
+        }));
+        const locatorHint = name
+          ? `select[multiple][name="${name}"]`
+          : el.id
+          ? `select[multiple]#${el.id}`
+          : 'select[multiple]';
+        return { label: labelText, locatorHint, options };
+      });
+
+      results.push({
+        kind: 'multiselect',
+        label: data.label,
+        locatorHint: data.locatorHint,
+        options: data.options,
+      });
+    } catch {
+      // skip
+    }
+  }
+
+  const ariaMulti = await page.$$('[role="listbox"][aria-multiselectable="true"]');
+  for (const handle of ariaMulti) {
+    try {
+      const data = await handle.evaluate((el: Element) => {
+        const ariaLabel = el.getAttribute('aria-label') || el.id || '';
+        const locatorHint = el.id
+          ? `[role="listbox"]#${el.id}`
+          : '[role="listbox"][aria-multiselectable="true"]';
+        const optionEls = el.querySelectorAll('[role="option"]');
+        const options = Array.from(optionEls).map((o) => ({
+          value: o.getAttribute('data-value') || o.getAttribute('value') || o.textContent?.trim() || '',
+          label: o.textContent?.trim() || '',
+          disabled: o.getAttribute('aria-disabled') === 'true' || undefined,
+        }));
+        return { label: ariaLabel, locatorHint, options };
+      });
+
+      results.push({
+        kind: 'multiselect',
+        label: data.label,
+        locatorHint: data.locatorHint,
+        options: data.options,
+      });
+    } catch {
+      // skip
+    }
+  }
+
+  return results;
+}
+
+async function detectCascading(
+  page: Page,
+  dropdowns: DropdownDef[]
+): Promise<Record<string, string[]>> {
+  const cascadeMap: Record<string, string[]> = {};
+
+  const nativeParents = dropdowns.filter(
+    (d) => d.kind === 'native-select' && d.options.length <= 8
+  );
+
+  for (const parent of nativeParents) {
+    const parentName = parent.locatorHint.match(/name="([^"]+)"/)?.[1];
+    if (!parentName) continue;
+
+    const otherNatives = dropdowns.filter(
+      (d) => d.kind === 'native-select' && d.locatorHint !== parent.locatorHint
+    );
+    if (otherNatives.length === 0) continue;
+
+    const baselineCounts = await page
+      .$$eval('select:not([multiple])', (els) =>
+        (els as HTMLSelectElement[]).map((el) => ({ name: el.name || el.id, count: el.options.length }))
+      )
+      .catch(() => [] as { name: string; count: number }[]);
+
+    let foundChildren: string[] = [];
+
+    for (let i = 0; i < parent.options.length; i++) {
+      try {
+        await page.selectOption(`select[name="${parentName}"]`, { index: i });
+        await page.waitForTimeout(500);
+
+        const newCounts = await page
+          .$$eval('select:not([multiple])', (els) =>
+            (els as HTMLSelectElement[]).map((el) => ({ name: el.name || el.id, count: el.options.length }))
+          )
+          .catch(() => [] as { name: string; count: number }[]);
+
+        for (const curr of newCounts) {
+          const base = baselineCounts.find((b) => b.name === curr.name);
+          if (base && base.count !== curr.count && curr.name !== parentName) {
+            const childHint = `select[name="${curr.name}"]`;
+            const childDef = dropdowns.find((d) => d.locatorHint === childHint);
+            if (childDef && !foundChildren.includes(childHint)) {
+              foundChildren.push(childHint);
+            }
+          }
+        }
+      } catch {
+        // skip option that fails
+      }
+    }
+
+    // reset to index 0
+    try {
+      await page.selectOption(`select[name="${parentName}"]`, { index: 0 });
+    } catch {
+      // ignore reset failure
+    }
+
+    if (foundChildren.length > 0) {
+      cascadeMap[parent.locatorHint] = foundChildren;
+    }
+  }
+
+  return cascadeMap;
+}
+
+export async function enumerateDropdowns(page: Page): Promise<DropdownEnumerationResult> {
+  try {
+    const [native, custom, multi] = await Promise.all([
+      enumerateNativeSelects(page).catch(() => [] as DropdownDef[]),
+      enumerateCustomDropdowns(page).catch(() => [] as DropdownDef[]),
+      enumerateMultiSelects(page).catch(() => [] as DropdownDef[]),
+    ]);
+
+    const seen = new Set<string>();
+    const dropdowns: DropdownDef[] = [];
+
+    for (const d of [...native, ...custom, ...multi]) {
+      if (!seen.has(d.locatorHint)) {
+        seen.add(d.locatorHint);
+        dropdowns.push(d);
+      }
+    }
+
+    const conditionalRevealMap = await detectCascading(page, dropdowns).catch(
+      () => ({} as Record<string, string[]>)
+    );
+
+    for (const [parent, children] of Object.entries(conditionalRevealMap)) {
+      for (const childHint of children) {
+        const child = dropdowns.find((d) => d.locatorHint === childHint);
+        if (child) child.cascadesFrom = parent;
+      }
+    }
+
+    const totalOptionsFound = dropdowns.reduce((sum, d) => sum + d.options.length, 0);
+
+    return { dropdowns, conditionalRevealMap, totalOptionsFound };
+  } catch {
+    return { dropdowns: [], conditionalRevealMap: {}, totalOptionsFound: 0 };
+  }
+}