@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/dropdown-enumerator.ts

@@ -0,0 +1,312 @@
+import type { Page } from 'playwright';
+
+export interface DropdownOption {
+  value: string;
+  label: string;
+  disabled?: boolean;
+}
+
+export interface DropdownDef {
+  kind: 'native-select' | 'combobox' | 'listbox' | 'multiselect';
+  label: string;
+  locatorHint: string;
+  options: DropdownOption[];
+  defaultValue?: string;
+  cascadesFrom?: string;
+}
+
+export interface DropdownEnumerationResult {
+  dropdowns: DropdownDef[];
+  conditionalRevealMap: Record<string, string[]>;
+  totalOptionsFound: number;
+}
+
+async function enumerateNativeSelects(page: Page): Promise<DropdownDef[]> {
+  const selects = await page.$$('select:not([multiple])');
+  const limited = selects.slice(0, 50);
+  const results: DropdownDef[] = [];
+
+  for (const handle of limited) {
+    try {
+      const data = await handle.evaluate((el: HTMLSelectElement) => {
+        const name = el.name || el.id || '';
+        const ariaLabel = el.getAttribute('aria-label') || '';
+        let labelText = ariaLabel;
+        if (!labelText && el.id) {
+          const lbl = document.querySelector(`label[for="${el.id}"]`);
+          if (lbl) labelText = lbl.textContent?.trim() || '';
+        }
+        if (!labelText && name) labelText = name;
+        const options = Array.from(el.options).map((o) => ({
+          value: o.value,
+          label: o.text.trim(),
+          disabled: o.disabled || undefined,
+        }));
+        const selectedOption = el.options[el.selectedIndex];
+        const defaultValue = selectedOption ? selectedOption.value : undefined;
+        const locatorHint = name
+          ? `select[name="${name}"]`
+          : el.id
+          ? `select#${el.id}`
+          : 'select';
+        return { label: labelText, locatorHint, options, defaultValue };
+      });
+
+      results.push({
+        kind: 'native-select',
+        label: data.label,
+        locatorHint: data.locatorHint,
+        options: data.options,
+        defaultValue: data.defaultValue,
+      });
+    } catch {
+      // skip unresponsive handle
+    }
+  }
+
+  return results;
+}
+
+async function enumerateCustomDropdowns(page: Page): Promise<DropdownDef[]> {
+  const selectors = [
+    '[role="combobox"]',
+    '[aria-haspopup="listbox"]:not([role="combobox"])',
+    '[data-radix-select-trigger]',
+  ];
+
+  const seen = new Set<string>();
+  const handles: { handle: Awaited<ReturnType<Page['$$']>>[0]; hint: string }[] = [];
+
+  for (const sel of selectors) {
+    const els = await page.$$(sel);
+    for (const el of els) {
+      const hint = await el
+        .evaluate((e: Element) => {
+          const id = e.id ? `#${e.id}` : '';
+          const aria = e.getAttribute('aria-label') || '';
+          return id || aria || e.tagName.toLowerCase();
+        })
+        .catch(() => '');
+      if (!seen.has(hint)) {
+        seen.add(hint);
+        handles.push({ handle: el, hint });
+      }
+    }
+  }
+
+  const limited = handles.slice(0, 20);
+  const results: DropdownDef[] = [];
+
+  for (const { handle, hint } of limited) {
+    try {
+      const ariaLabel = await handle
+        .evaluate((e: Element) => e.getAttribute('aria-label') || e.textContent?.trim() || '')
+        .catch(() => '');
+
+      await handle.click();
+      await page.waitForSelector('[role="listbox"]', { timeout: 2000 });
+
+      const options = await page.$$eval('[role="listbox"] [role="option"]', (els) =>
+        els.map((el) => ({
+          value: el.getAttribute('data-value') || el.getAttribute('value') || el.textContent?.trim() || '',
+          label: el.textContent?.trim() || '',
+          disabled: el.getAttribute('aria-disabled') === 'true' || undefined,
+        }))
+      );
+
+      await page.keyboard.press('Escape');
+      await page.waitForTimeout(300);
+
+      results.push({
+        kind: 'combobox',
+        label: ariaLabel,
+        locatorHint: hint,
+        options,
+      });
+    } catch {
+      try {
+        await page.keyboard.press('Escape');
+        await page.waitForTimeout(300);
+      } catch {
+        // ignore cleanup failure
+      }
+    }
+  }
+
+  return results;
+}
+
+async function enumerateMultiSelects(page: Page): Promise<DropdownDef[]> {
+  const results: DropdownDef[] = [];
+
+  const nativeMulti = await page.$$('select[multiple]');
+  for (const handle of nativeMulti) {
+    try {
+      const data = await handle.evaluate((el: HTMLSelectElement) => {
+        const name = el.name || el.id || '';
+        const ariaLabel = el.getAttribute('aria-label') || '';
+        let labelText = ariaLabel;
+        if (!labelText && el.id) {
+          const lbl = document.querySelector(`label[for="${el.id}"]`);
+          if (lbl) labelText = lbl.textContent?.trim() || '';
+        }
+        if (!labelText && name) labelText = name;
+        const options = Array.from(el.options).map((o) => ({
+          value: o.value,
+          label: o.text.trim(),
+          disabled: o.disabled || undefined,
+        }));
+        const locatorHint = name
+          ? `select[multiple][name="${name}"]`
+          : el.id
+          ? `select[multiple]#${el.id}`
+          : 'select[multiple]';
+        return { label: labelText, locatorHint, options };
+      });
+
+      results.push({
+        kind: 'multiselect',
+        label: data.label,
+        locatorHint: data.locatorHint,
+        options: data.options,
+      });
+    } catch {
+      // skip
+    }
+  }
+
+  const ariaMulti = await page.$$('[role="listbox"][aria-multiselectable="true"]');
+  for (const handle of ariaMulti) {
+    try {
+      const data = await handle.evaluate((el: Element) => {
+        const ariaLabel = el.getAttribute('aria-label') || el.id || '';
+        const locatorHint = el.id
+          ? `[role="listbox"]#${el.id}`
+          : '[role="listbox"][aria-multiselectable="true"]';
+        const optionEls = el.querySelectorAll('[role="option"]');
+        const options = Array.from(optionEls).map((o) => ({
+          value: o.getAttribute('data-value') || o.getAttribute('value') || o.textContent?.trim() || '',
+          label: o.textContent?.trim() || '',
+          disabled: o.getAttribute('aria-disabled') === 'true' || undefined,
+        }));
+        return { label: ariaLabel, locatorHint, options };
+      });
+
+      results.push({
+        kind: 'multiselect',
+        label: data.label,
+        locatorHint: data.locatorHint,
+        options: data.options,
+      });
+    } catch {
+      // skip
+    }
+  }
+
+  return results;
+}
+
+async function detectCascading(
+  page: Page,
+  dropdowns: DropdownDef[]
+): Promise<Record<string, string[]>> {
+  const cascadeMap: Record<string, string[]> = {};
+
+  const nativeParents = dropdowns.filter(
+    (d) => d.kind === 'native-select' && d.options.length <= 8
+  );
+
+  for (const parent of nativeParents) {
+    const parentName = parent.locatorHint.match(/name="([^"]+)"/)?.[1];
+    if (!parentName) continue;
+
+    const otherNatives = dropdowns.filter(
+      (d) => d.kind === 'native-select' && d.locatorHint !== parent.locatorHint
+    );
+    if (otherNatives.length === 0) continue;
+
+    const baselineCounts = await page
+      .$$eval('select:not([multiple])', (els) =>
+        (els as HTMLSelectElement[]).map((el) => ({ name: el.name || el.id, count: el.options.length }))
+      )
+      .catch(() => [] as { name: string; count: number }[]);
+
+    let foundChildren: string[] = [];
+
+    for (let i = 0; i < parent.options.length; i++) {
+      try {
+        await page.selectOption(`select[name="${parentName}"]`, { index: i });
+        await page.waitForTimeout(500);
+
+        const newCounts = await page
+          .$$eval('select:not([multiple])', (els) =>
+            (els as HTMLSelectElement[]).map((el) => ({ name: el.name || el.id, count: el.options.length }))
+          )
+          .catch(() => [] as { name: string; count: number }[]);
+
+        for (const curr of newCounts) {
+          const base = baselineCounts.find((b) => b.name === curr.name);
+          if (base && base.count !== curr.count && curr.name !== parentName) {
+            const childHint = `select[name="${curr.name}"]`;
+            const childDef = dropdowns.find((d) => d.locatorHint === childHint);
+            if (childDef && !foundChildren.includes(childHint)) {
+              foundChildren.push(childHint);
+            }
+          }
+        }
+      } catch {
+        // skip option that fails
+      }
+    }
+
+    // reset to index 0
+    try {
+      await page.selectOption(`select[name="${parentName}"]`, { index: 0 });
+    } catch {
+      // ignore reset failure
+    }
+
+    if (foundChildren.length > 0) {
+      cascadeMap[parent.locatorHint] = foundChildren;
+    }
+  }
+
+  return cascadeMap;
+}
+
+export async function enumerateDropdowns(page: Page): Promise<DropdownEnumerationResult> {
+  try {
+    const [native, custom, multi] = await Promise.all([
+      enumerateNativeSelects(page).catch(() => [] as DropdownDef[]),
+      enumerateCustomDropdowns(page).catch(() => [] as DropdownDef[]),
+      enumerateMultiSelects(page).catch(() => [] as DropdownDef[]),
+    ]);
+
+    const seen = new Set<string>();
+    const dropdowns: DropdownDef[] = [];
+
+    for (const d of [...native, ...custom, ...multi]) {
+      if (!seen.has(d.locatorHint)) {
+        seen.add(d.locatorHint);
+        dropdowns.push(d);
+      }
+    }
+
+    const conditionalRevealMap = await detectCascading(page, dropdowns).catch(
+      () => ({} as Record<string, string[]>)
+    );
+
+    for (const [parent, children] of Object.entries(conditionalRevealMap)) {
+      for (const childHint of children) {
+        const child = dropdowns.find((d) => d.locatorHint === childHint);
+        if (child) child.cascadesFrom = parent;
+      }
+    }
+
+    const totalOptionsFound = dropdowns.reduce((sum, d) => sum + d.options.length, 0);
+
+    return { dropdowns, conditionalRevealMap, totalOptionsFound };
+  } catch {
+    return { dropdowns: [], conditionalRevealMap: {}, totalOptionsFound: 0 };
+  }
+}

package/src/element-hash.ts

@@ -0,0 +1,16 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Compute a stable fingerprint of a screen's element set.
+ * Sorted so insertion-order changes (React re-renders) don't produce false positives.
+ * Returns first 16 hex chars of SHA-256.
+ */
+export function computeElementHash(
+  elements: Array<{ role: string; meaning: string }>,
+): string {
+  const pairs = elements
+    .map((e) => `${e.role.toLowerCase()}:${e.meaning.toLowerCase().trim()}`)
+    .sort()
+    .join('|');
+  return createHash('sha256').update(pairs).digest('hex').slice(0, 16);
+}

package/src/email-otp.ts

@@ -0,0 +1,252 @@
+/**
+ * Email OTP retrieval — supports 3 providers:
+ *   mailslurp  — managed SaaS (existing, no self-hosting)
+ *   imap       — any real IMAP mailbox (Gmail, Outlook, custom domain catch-all)
+ *   mailpit    — self-hosted SMTP trap (dev/staging only)
+ *
+ * Use mailslurp or imap for production apps.
+ * Use mailpit only when you can point the app's SMTP at the trap server.
+ */
+
+export type EmailOTPProvider = 'mailslurp' | 'imap' | 'mailpit';
+
+export interface EmailContent {
+  /** 4-8 digit OTP code, if found. */
+  otp?: string;
+  /** Magic link URL to navigate to, if found. */
+  magicLink?: string;
+}
+
+export interface EmailOTPConfig {
+  provider: EmailOTPProvider;
+  // MailSlurp
+  apiKey?: string;
+  inboxId?: string;
+  // IMAP (Gmail app-password, Outlook, custom IMAP)
+  imapHost?: string;
+  imapPort?: number;
+  imapUser?: string;
+  imapPass?: string;
+  imapTls?: boolean;
+  // Mailpit (REST API at baseUrl)
+  mailpitBaseUrl?: string;
+}
+
+const POLL_INTERVAL_MS = 2_000;
+const MAX_WAIT_MS = 60_000;
+
+/** Extract OTP code from email body text. Prefers 6-digit, falls back to 4-8 digit. */
+function extractOTP(text: string): string | null {
+  const m6 = text.match(/\b(\d{6})\b/);
+  if (m6) return m6[1];
+  const m48 = text.match(/\b(\d{4,8})\b/);
+  if (m48) return m48[1];
+  const magic = text.match(/\b([A-Z0-9]{6,10})\b/);
+  if (magic) return magic[1];
+  return null;
+}
+
+/** Extract a magic-link URL from email body text. Matches /verify, /magic, /auth/callback, ?token= etc. */
+function extractMagicLink(text: string): string | null {
+  const magicPath = /\/(verify|magic|login|auth|confirm|sign-in)[/?#]/i;
+  const magicQuery = /[?&](token|magic_token|login_token|auth_token|key)=/i;
+  const urlMatches = text.match(/https:\/\/[^\s<>"')\]]+/g);
+  if (!urlMatches) return null;
+  for (const raw of urlMatches) {
+    try {
+      const u = new URL(raw.replace(/[>'")\].,]+$/, ''));
+      if (magicPath.test(u.pathname) || magicQuery.test(u.search)) return u.toString();
+    } catch { /* skip malformed URLs */ }
+  }
+  return null;
+}
+
+function parseEmailContent(text: string): EmailContent {
+  const magicLink = extractMagicLink(text);
+  if (magicLink) return { magicLink };
+  const otp = extractOTP(text);
+  if (otp) return { otp };
+  return {};
+}
+
+// ── MailSlurp ────────────────────────────────────────────────────────────────
+
+const MAILSLURP_BASE = 'https://api.mailslurp.com';
+
+async function waitForContentMailSlurp(apiKey: string, inboxId: string, since: string): Promise<EmailContent | null> {
+  const deadline = Date.now() + MAX_WAIT_MS;
+  while (Date.now() < deadline) {
+    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+    try {
+      const listRes = await fetch(
+        `${MAILSLURP_BASE}/inboxes/${inboxId}/emails?sort=DESC&limit=5&since=${encodeURIComponent(since)}`,
+        { headers: { 'x-api-key': apiKey } }
+      );
+      if (!listRes.ok) continue;
+      const emails = await listRes.json() as Array<{ id: string }>;
+      if (!emails.length) continue;
+      const bodyRes = await fetch(`${MAILSLURP_BASE}/emails/${emails[0].id}`, { headers: { 'x-api-key': apiKey } });
+      if (!bodyRes.ok) continue;
+      const email = await bodyRes.json() as { body?: string; subject?: string };
+      const content = parseEmailContent((email.body ?? '') + ' ' + (email.subject ?? ''));
+      if (content.otp || content.magicLink) return content;
+    } catch { /* retry */ }
+  }
+  return null;
+}
+
+async function getInboxEmailMailSlurp(apiKey: string, inboxId: string): Promise<string | null> {
+  try {
+    const res = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}`, { headers: { 'x-api-key': apiKey } });
+    if (!res.ok) return null;
+    const data = await res.json() as { emailAddress?: string };
+    return data.emailAddress ?? null;
+  } catch { return null; }
+}
+
+// ── Mailpit (SMTP trap REST API) ──────────────────────────────────────────────
+
+async function waitForContentMailpit(baseUrl: string, since: Date): Promise<EmailContent | null> {
+  const deadline = Date.now() + MAX_WAIT_MS;
+  const sinceMs = since.getTime();
+  while (Date.now() < deadline) {
+    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+    try {
+      const res = await fetch(`${baseUrl}/api/v1/messages?limit=10`);
+      if (!res.ok) continue;
+      const data = await res.json() as { messages?: Array<{ ID: string; Created: string; Snippet: string; Subject: string }> };
+      const recent = (data.messages ?? []).filter(m => new Date(m.Created).getTime() > sinceMs);
+      for (const msg of recent) {
+        const snippet = parseEmailContent(msg.Snippet + ' ' + msg.Subject);
+        if (snippet.otp || snippet.magicLink) return snippet;
+        // Fetch full body — snippet may truncate the magic link
+        const bodyRes = await fetch(`${baseUrl}/api/v1/message/${msg.ID}`);
+        if (!bodyRes.ok) continue;
+        const body = await bodyRes.json() as { Text?: { Body?: string }; Subject?: string };
+        const full = parseEmailContent((body.Text?.Body ?? '') + ' ' + (body.Subject ?? ''));
+        if (full.otp || full.magicLink) return full;
+      }
+    } catch { /* retry */ }
+  }
+  return null;
+}
+
+// ── IMAP (any mailbox) ────────────────────────────────────────────────────────
+
+async function waitForContentImap(
+  cfg: { host: string; port: number; user: string; pass: string; tls: boolean },
+  since: Date
+): Promise<EmailContent | null> {
+  let ImapFlow: any;
+  try {
+    const mod = await import('imapflow');
+    ImapFlow = mod.ImapFlow;
+  } catch {
+    throw new Error('IMAP provider requires imapflow package. Add it to dependencies.');
+  }
+
+  const client = new ImapFlow({
+    host: cfg.host,
+    port: cfg.port,
+    secure: cfg.tls,
+    auth: { user: cfg.user, pass: cfg.pass },
+    logger: false,
+  });
+
+  try {
+    await client.connect();
+    const lock = await client.getMailboxLock('INBOX');
+    try {
+      const deadline = Date.now() + MAX_WAIT_MS;
+      while (Date.now() < deadline) {
+        const uids = await client.search({ seen: false, since });
+        if (uids.length) {
+          for await (const msg of client.fetch(uids.slice(-5), { source: true })) {
+            const { simpleParser } = await import('mailparser');
+            const parsed = await simpleParser(msg.source);
+            const text = (parsed.text ?? '') + ' ' + (parsed.subject ?? '');
+            const content = parseEmailContent(text);
+            if (content.otp || content.magicLink) return content;
+          }
+        }
+        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+      }
+      return null;
+    } finally {
+      lock.release();
+    }
+  } finally {
+    await client.logout().catch(() => {});
+  }
+}
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+/**
+ * Wait for an email and return its OTP code or magic link URL.
+ * Prefer this over waitForEmailOTP when the app may send passwordless magic links.
+ */
+export async function waitForEmailContent(
+  config: EmailOTPConfig,
+  since: string | Date
+): Promise<EmailContent | null> {
+  const sinceStr = typeof since === 'string' ? since : since.toISOString();
+  const sinceDate = typeof since === 'string' ? new Date(since) : since;
+
+  switch (config.provider) {
+    case 'mailslurp':
+      if (!config.apiKey || !config.inboxId) throw new Error('mailslurp requires apiKey + inboxId');
+      return waitForContentMailSlurp(config.apiKey, config.inboxId, sinceStr);
+
+    case 'mailpit':
+      return waitForContentMailpit(config.mailpitBaseUrl ?? 'http://mailpit:8025', sinceDate);
+
+    case 'imap':
+      if (!config.imapHost || !config.imapUser || !config.imapPass) {
+        throw new Error('imap requires imapHost, imapUser, imapPass');
+      }
+      return waitForContentImap({
+        host: config.imapHost,
+        port: config.imapPort ?? 993,
+        user: config.imapUser,
+        pass: config.imapPass,
+        tls: config.imapTls !== false,
+      }, sinceDate);
+
+    default:
+      return null;
+  }
+}
+
+/**
+ * Wait for an OTP email and extract the code.
+ * @deprecated Use waitForEmailContent which also handles magic links.
+ */
+export async function waitForEmailOTP(
+  config: EmailOTPConfig,
+  since: string | Date
+): Promise<string | null> {
+  const content = await waitForEmailContent(config, since);
+  return content?.otp ?? null;
+}
+
+/**
+ * Get the email address for a MailSlurp inbox (used to display in UI).
+ * Returns null for IMAP/Mailpit providers — use imapUser directly.
+ */
+export async function getInboxEmail(
+  configOrApiKey: EmailOTPConfig | string,
+  inboxId?: string
+): Promise<string | null> {
+  // Legacy call: getInboxEmail(apiKey, inboxId)
+  if (typeof configOrApiKey === 'string') {
+    return getInboxEmailMailSlurp(configOrApiKey, inboxId ?? '');
+  }
+  if (configOrApiKey.provider === 'mailslurp' && configOrApiKey.apiKey && configOrApiKey.inboxId) {
+    return getInboxEmailMailSlurp(configOrApiKey.apiKey, configOrApiKey.inboxId);
+  }
+  if (configOrApiKey.provider === 'imap') {
+    return configOrApiKey.imapUser ?? null;
+  }
+  return null;
+}