@detiq/crawler 0.1.3

package/src/custom-js-injector.ts

@@ -0,0 +1,93 @@
+/**
+ * Custom JS injector — run user-provided JavaScript in the context of each crawled page.
+ *
+ * Use cases:
+ * - Extract custom data points not captured by standard extractors
+ * - Set localStorage/sessionStorage values before crawl
+ * - Override fetch() to intercept API calls
+ * - Simulate user events (keydown, focus, etc.)
+ * - Bypass specific anti-bot checks
+ *
+ * Security: scripts run in the page context (sandboxed by Chromium's renderer).
+ * They cannot access the Node.js process or file system.
+ * ZeTa validates scripts are strings before injection.
+ */
+
+import type { Page } from 'playwright';
+
+export interface JsInjectionOpts {
+  /** Script to run on every page BEFORE navigation (via page.addInitScript) */
+  initScript?: string;
+  /** Script to run on every page AFTER navigation completes (via page.evaluate) */
+  postLoadScript?: string;
+  /** Key-value pairs to set in localStorage before page load */
+  localStorage?: Record<string, string>;
+  /** Key-value pairs to set in sessionStorage before page load */
+  sessionStorage?: Record<string, string>;
+  /** Cookie key-value pairs to add before page load */
+  cookies?: Array<{ name: string; value: string; domain?: string; path?: string }>;
+}
+
+export interface JsInjectionResult {
+  initScriptApplied: boolean;
+  postLoadResult: unknown;
+  storageSet: boolean;
+  cookiesSet: number;
+  error?: string;
+}
+
+export async function injectCustomJs(page: Page, opts: JsInjectionOpts): Promise<JsInjectionResult> {
+  const result: JsInjectionResult = {
+    initScriptApplied: false,
+    postLoadResult: null,
+    storageSet: false,
+    cookiesSet: 0,
+  };
+
+  // Set cookies via Playwright context
+  if (opts.cookies?.length) {
+    try {
+      await page.context().addCookies(
+        opts.cookies.map(c => ({ name: c.name, value: c.value, domain: c.domain ?? '', path: c.path ?? '/', url: page.url() || undefined }))
+      );
+      result.cookiesSet = opts.cookies.length;
+    } catch (e: any) {
+      result.error = `Cookie injection failed: ${e.message}`;
+    }
+  }
+
+  // Init script (runs before page load on every navigation)
+  if (opts.initScript || opts.localStorage || opts.sessionStorage) {
+    const storageSetup = [
+      opts.localStorage ? `Object.entries(${JSON.stringify(opts.localStorage)}).forEach(([k,v])=>localStorage.setItem(k,v));` : '',
+      opts.sessionStorage ? `Object.entries(${JSON.stringify(opts.sessionStorage)}).forEach(([k,v])=>sessionStorage.setItem(k,v));` : '',
+    ].filter(Boolean).join('\n');
+
+    const combinedInit = [storageSetup, opts.initScript ?? ''].filter(Boolean).join('\n');
+    try {
+      await page.addInitScript(combinedInit);
+      result.initScriptApplied = true;
+      result.storageSet = !!(opts.localStorage || opts.sessionStorage);
+    } catch (e: any) {
+      result.error = `Init script failed: ${e.message}`;
+    }
+  }
+
+  // Post-load script (runs after navigation)
+  if (opts.postLoadScript) {
+    try {
+      result.postLoadResult = await page.evaluate(opts.postLoadScript);
+    } catch (e: any) {
+      result.error = `Post-load script failed: ${e.message}`;
+    }
+  }
+
+  return result;
+}
+
+/** Safe string validator — ensures script is a string, not an object/function reference */
+export function validateScript(script: unknown): string {
+  if (typeof script !== 'string') throw new Error('Script must be a string');
+  if (script.length > 50_000) throw new Error('Script too large (>50KB)');
+  return script;
+}

package/src/dark-mode-capture.ts

@@ -0,0 +1,110 @@
+/**
+ * Dark mode capture — screenshot pages in both light and dark color schemes.
+ *
+ * Uses Playwright's page.emulateMedia({ colorScheme: 'dark' }) to force dark mode,
+ * then captures a screenshot. Compares with light mode to determine if dark mode
+ * is supported (significant visual difference).
+ *
+ * Also detects prefers-color-scheme media query support in CSS.
+ */
+
+import type { Page } from 'playwright';
+
+export interface DarkModeCapture {
+  url: string;
+  lightScreenshot?: Buffer;
+  darkScreenshot?: Buffer;
+  hasDarkMode: boolean;
+  hasCssDarkMode: boolean; // @media (prefers-color-scheme: dark) detected in stylesheets
+  backgroundColorLight?: string;
+  backgroundColorDark?: string;
+  capturedAt: string;
+}
+
+async function getBackgroundColor(page: Page): Promise<string> {
+  try {
+    return await page.evaluate(() => {
+      const body = document.body;
+      return window.getComputedStyle(body).backgroundColor;
+    });
+  } catch {
+    return 'unknown';
+  }
+}
+
+function colorsAreDifferent(a: string, b: string): boolean {
+  if (a === b) return false;
+  if (a === 'unknown' || b === 'unknown') return false;
+  // Parse rgb(r, g, b) and compare luminance
+  const parse = (c: string) => {
+    const m = c.match(/\d+/g);
+    return m ? m.map(Number) : [0, 0, 0];
+  };
+  const [ar, ag, ab] = parse(a);
+  const [br, bg, bb] = parse(b);
+  const lumA = 0.299 * ar + 0.587 * ag + 0.114 * ab;
+  const lumB = 0.299 * br + 0.587 * bg + 0.114 * bb;
+  return Math.abs(lumA - lumB) > 30; // significant luminance difference
+}
+
+async function hasCssDarkModeSupport(page: Page): Promise<boolean> {
+  try {
+    return await page.evaluate(() => {
+      const sheets = Array.from(document.styleSheets);
+      for (const sheet of sheets) {
+        try {
+          const rules = Array.from(sheet.cssRules ?? []);
+          for (const rule of rules) {
+            if (rule instanceof CSSMediaRule && rule.conditionText?.includes('prefers-color-scheme')) {
+              return true;
+            }
+          }
+        } catch { /* cross-origin stylesheet — skip */ }
+      }
+      return false;
+    });
+  } catch {
+    return false;
+  }
+}
+
+export async function captureDarkMode(
+  page: Page,
+  url: string,
+  opts: { takeScreenshots?: boolean } = {},
+): Promise<DarkModeCapture> {
+  const capturedAt = new Date().toISOString();
+  const { takeScreenshots = false } = opts;
+
+  // Light mode
+  await page.emulateMedia({ colorScheme: 'light' }).catch(() => {});
+  const backgroundColorLight = await getBackgroundColor(page);
+  const hasCssDark = await hasCssDarkModeSupport(page);
+  const lightScreenshot = takeScreenshots
+    ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
+    : undefined;
+
+  // Dark mode
+  await page.emulateMedia({ colorScheme: 'dark' }).catch(() => {});
+  await page.waitForTimeout(300); // allow CSS transitions
+  const backgroundColorDark = await getBackgroundColor(page);
+  const darkScreenshot = takeScreenshots
+    ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
+    : undefined;
+
+  // Restore light mode
+  await page.emulateMedia({ colorScheme: 'light' }).catch(() => {});
+
+  const hasDarkMode = colorsAreDifferent(backgroundColorLight, backgroundColorDark) || hasCssDark;
+
+  return {
+    url,
+    lightScreenshot,
+    darkScreenshot,
+    hasDarkMode,
+    hasCssDarkMode: hasCssDark,
+    backgroundColorLight,
+    backgroundColorDark,
+    capturedAt,
+  };
+}

package/src/dead-link-detector.ts

@@ -0,0 +1,76 @@
+/**
+ * Dead link detector — tracks broken links discovered during crawl.
+ *
+ * A "dead link" is a URL that:
+ * - Returns 404, 410, or other 4xx/5xx responses
+ * - Times out after maxRetries
+ * - Redirects to an error page (detected by title heuristic)
+ *
+ * Results: list of broken links with source page, status code, and anchor text.
+ * Used for: SEO audit export, test report enrichment.
+ */
+
+export interface DeadLink {
+  url: string;
+  statusCode: number;
+  sourceUrl?: string;
+  anchorText?: string;
+  checkedAt: string;
+  reason: 'http-error' | 'timeout' | 'dns-error';
+}
+
+export class DeadLinkTracker {
+  private deadLinks: DeadLink[] = [];
+  private checked = new Map<string, number>(); // url → statusCode
+
+  record(
+    url: string,
+    statusCode: number,
+    source?: { sourceUrl?: string; anchorText?: string },
+  ): void {
+    if (this.checked.has(url)) return;
+    this.checked.set(url, statusCode);
+    if (statusCode >= 400) {
+      this.deadLinks.push({
+        url,
+        statusCode,
+        sourceUrl: source?.sourceUrl,
+        anchorText: source?.anchorText,
+        checkedAt: new Date().toISOString(),
+        reason: statusCode === 0 ? 'timeout' : 'http-error',
+      });
+    }
+  }
+
+  recordTimeout(url: string, sourceUrl?: string): void {
+    if (this.checked.has(url)) return;
+    this.checked.set(url, 0);
+    this.deadLinks.push({ url, statusCode: 0, sourceUrl, checkedAt: new Date().toISOString(), reason: 'timeout' });
+  }
+
+  getDeadLinks(): DeadLink[] {
+    return [...this.deadLinks];
+  }
+
+  isKnownBroken(url: string): boolean {
+    const code = this.checked.get(url);
+    return code !== undefined && code >= 400;
+  }
+
+  stats(): { total: number; by404: number; by5xx: number; byTimeout: number } {
+    return {
+      total: this.deadLinks.length,
+      by404: this.deadLinks.filter(l => l.statusCode === 404).length,
+      by5xx: this.deadLinks.filter(l => l.statusCode >= 500).length,
+      byTimeout: this.deadLinks.filter(l => l.reason === 'timeout').length,
+    };
+  }
+
+  toCsv(): string {
+    const rows = [['URL', 'Status', 'Source URL', 'Anchor Text', 'Checked At']];
+    for (const l of this.deadLinks) {
+      rows.push([l.url, String(l.statusCode), l.sourceUrl ?? '', l.anchorText ?? '', l.checkedAt]);
+    }
+    return rows.map(r => r.map(c => `"${c.replace(/"/g, '""')}"`).join(',')).join('\n');
+  }
+}

package/src/element-hash.ts

@@ -0,0 +1,16 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Compute a stable fingerprint of a screen's element set.
+ * Sorted so insertion-order changes (React re-renders) don't produce false positives.
+ * Returns first 16 hex chars of SHA-256.
+ */
+export function computeElementHash(
+  elements: Array<{ role: string; meaning: string }>,
+): string {
+  const pairs = elements
+    .map((e) => `${e.role.toLowerCase()}:${e.meaning.toLowerCase().trim()}`)
+    .sort()
+    .join('|');
+  return createHash('sha256').update(pairs).digest('hex').slice(0, 16);
+}

package/src/email-otp.ts

@@ -0,0 +1,252 @@
+/**
+ * Email OTP retrieval — supports 3 providers:
+ *   mailslurp  — managed SaaS (existing, no self-hosting)
+ *   imap       — any real IMAP mailbox (Gmail, Outlook, custom domain catch-all)
+ *   mailpit    — self-hosted SMTP trap (dev/staging only)
+ *
+ * Use mailslurp or imap for production apps.
+ * Use mailpit only when you can point the app's SMTP at the trap server.
+ */
+
+export type EmailOTPProvider = 'mailslurp' | 'imap' | 'mailpit';
+
+export interface EmailContent {
+  /** 4-8 digit OTP code, if found. */
+  otp?: string;
+  /** Magic link URL to navigate to, if found. */
+  magicLink?: string;
+}
+
+export interface EmailOTPConfig {
+  provider: EmailOTPProvider;
+  // MailSlurp
+  apiKey?: string;
+  inboxId?: string;
+  // IMAP (Gmail app-password, Outlook, custom IMAP)
+  imapHost?: string;
+  imapPort?: number;
+  imapUser?: string;
+  imapPass?: string;
+  imapTls?: boolean;
+  // Mailpit (REST API at baseUrl)
+  mailpitBaseUrl?: string;
+}
+
+const POLL_INTERVAL_MS = 2_000;
+const MAX_WAIT_MS = 60_000;
+
+/** Extract OTP code from email body text. Prefers 6-digit, falls back to 4-8 digit. */
+function extractOTP(text: string): string | null {
+  const m6 = text.match(/\b(\d{6})\b/);
+  if (m6) return m6[1];
+  const m48 = text.match(/\b(\d{4,8})\b/);
+  if (m48) return m48[1];
+  const magic = text.match(/\b([A-Z0-9]{6,10})\b/);
+  if (magic) return magic[1];
+  return null;
+}
+
+/** Extract a magic-link URL from email body text. Matches /verify, /magic, /auth/callback, ?token= etc. */
+function extractMagicLink(text: string): string | null {
+  const magicPath = /\/(verify|magic|login|auth|confirm|sign-in)[/?#]/i;
+  const magicQuery = /[?&](token|magic_token|login_token|auth_token|key)=/i;
+  const urlMatches = text.match(/https:\/\/[^\s<>"')\]]+/g);
+  if (!urlMatches) return null;
+  for (const raw of urlMatches) {
+    try {
+      const u = new URL(raw.replace(/[>'")\].,]+$/, ''));
+      if (magicPath.test(u.pathname) || magicQuery.test(u.search)) return u.toString();
+    } catch { /* skip malformed URLs */ }
+  }
+  return null;
+}
+
+function parseEmailContent(text: string): EmailContent {
+  const magicLink = extractMagicLink(text);
+  if (magicLink) return { magicLink };
+  const otp = extractOTP(text);
+  if (otp) return { otp };
+  return {};
+}
+
+// ── MailSlurp ────────────────────────────────────────────────────────────────
+
+const MAILSLURP_BASE = 'https://api.mailslurp.com';
+
+async function waitForContentMailSlurp(apiKey: string, inboxId: string, since: string): Promise<EmailContent | null> {
+  const deadline = Date.now() + MAX_WAIT_MS;
+  while (Date.now() < deadline) {
+    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+    try {
+      const listRes = await fetch(
+        `${MAILSLURP_BASE}/inboxes/${inboxId}/emails?sort=DESC&limit=5&since=${encodeURIComponent(since)}`,
+        { headers: { 'x-api-key': apiKey } }
+      );
+      if (!listRes.ok) continue;
+      const emails = await listRes.json() as Array<{ id: string }>;
+      if (!emails.length) continue;
+      const bodyRes = await fetch(`${MAILSLURP_BASE}/emails/${emails[0].id}`, { headers: { 'x-api-key': apiKey } });
+      if (!bodyRes.ok) continue;
+      const email = await bodyRes.json() as { body?: string; subject?: string };
+      const content = parseEmailContent((email.body ?? '') + ' ' + (email.subject ?? ''));
+      if (content.otp || content.magicLink) return content;
+    } catch { /* retry */ }
+  }
+  return null;
+}
+
+async function getInboxEmailMailSlurp(apiKey: string, inboxId: string): Promise<string | null> {
+  try {
+    const res = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}`, { headers: { 'x-api-key': apiKey } });
+    if (!res.ok) return null;
+    const data = await res.json() as { emailAddress?: string };
+    return data.emailAddress ?? null;
+  } catch { return null; }
+}
+
+// ── Mailpit (SMTP trap REST API) ──────────────────────────────────────────────
+
+async function waitForContentMailpit(baseUrl: string, since: Date): Promise<EmailContent | null> {
+  const deadline = Date.now() + MAX_WAIT_MS;
+  const sinceMs = since.getTime();
+  while (Date.now() < deadline) {
+    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+    try {
+      const res = await fetch(`${baseUrl}/api/v1/messages?limit=10`);
+      if (!res.ok) continue;
+      const data = await res.json() as { messages?: Array<{ ID: string; Created: string; Snippet: string; Subject: string }> };
+      const recent = (data.messages ?? []).filter(m => new Date(m.Created).getTime() > sinceMs);
+      for (const msg of recent) {
+        const snippet = parseEmailContent(msg.Snippet + ' ' + msg.Subject);
+        if (snippet.otp || snippet.magicLink) return snippet;
+        // Fetch full body — snippet may truncate the magic link
+        const bodyRes = await fetch(`${baseUrl}/api/v1/message/${msg.ID}`);
+        if (!bodyRes.ok) continue;
+        const body = await bodyRes.json() as { Text?: { Body?: string }; Subject?: string };
+        const full = parseEmailContent((body.Text?.Body ?? '') + ' ' + (body.Subject ?? ''));
+        if (full.otp || full.magicLink) return full;
+      }
+    } catch { /* retry */ }
+  }
+  return null;
+}
+
+// ── IMAP (any mailbox) ────────────────────────────────────────────────────────
+
+async function waitForContentImap(
+  cfg: { host: string; port: number; user: string; pass: string; tls: boolean },
+  since: Date
+): Promise<EmailContent | null> {
+  let ImapFlow: any;
+  try {
+    const mod = await import('imapflow');
+    ImapFlow = mod.ImapFlow;
+  } catch {
+    throw new Error('IMAP provider requires imapflow package. Add it to dependencies.');
+  }
+
+  const client = new ImapFlow({
+    host: cfg.host,
+    port: cfg.port,
+    secure: cfg.tls,
+    auth: { user: cfg.user, pass: cfg.pass },
+    logger: false,
+  });
+
+  try {
+    await client.connect();
+    const lock = await client.getMailboxLock('INBOX');
+    try {
+      const deadline = Date.now() + MAX_WAIT_MS;
+      while (Date.now() < deadline) {
+        const uids = await client.search({ seen: false, since });
+        if (uids.length) {
+          for await (const msg of client.fetch(uids.slice(-5), { source: true })) {
+            const { simpleParser } = await import('mailparser');
+            const parsed = await simpleParser(msg.source);
+            const text = (parsed.text ?? '') + ' ' + (parsed.subject ?? '');
+            const content = parseEmailContent(text);
+            if (content.otp || content.magicLink) return content;
+          }
+        }
+        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+      }
+      return null;
+    } finally {
+      lock.release();
+    }
+  } finally {
+    await client.logout().catch(() => {});
+  }
+}
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+/**
+ * Wait for an email and return its OTP code or magic link URL.
+ * Prefer this over waitForEmailOTP when the app may send passwordless magic links.
+ */
+export async function waitForEmailContent(
+  config: EmailOTPConfig,
+  since: string | Date
+): Promise<EmailContent | null> {
+  const sinceStr = typeof since === 'string' ? since : since.toISOString();
+  const sinceDate = typeof since === 'string' ? new Date(since) : since;
+
+  switch (config.provider) {
+    case 'mailslurp':
+      if (!config.apiKey || !config.inboxId) throw new Error('mailslurp requires apiKey + inboxId');
+      return waitForContentMailSlurp(config.apiKey, config.inboxId, sinceStr);
+
+    case 'mailpit':
+      return waitForContentMailpit(config.mailpitBaseUrl ?? 'http://mailpit:8025', sinceDate);
+
+    case 'imap':
+      if (!config.imapHost || !config.imapUser || !config.imapPass) {
+        throw new Error('imap requires imapHost, imapUser, imapPass');
+      }
+      return waitForContentImap({
+        host: config.imapHost,
+        port: config.imapPort ?? 993,
+        user: config.imapUser,
+        pass: config.imapPass,
+        tls: config.imapTls !== false,
+      }, sinceDate);
+
+    default:
+      return null;
+  }
+}
+
+/**
+ * Wait for an OTP email and extract the code.
+ * @deprecated Use waitForEmailContent which also handles magic links.
+ */
+export async function waitForEmailOTP(
+  config: EmailOTPConfig,
+  since: string | Date
+): Promise<string | null> {
+  const content = await waitForEmailContent(config, since);
+  return content?.otp ?? null;
+}
+
+/**
+ * Get the email address for a MailSlurp inbox (used to display in UI).
+ * Returns null for IMAP/Mailpit providers — use imapUser directly.
+ */
+export async function getInboxEmail(
+  configOrApiKey: EmailOTPConfig | string,
+  inboxId?: string
+): Promise<string | null> {
+  // Legacy call: getInboxEmail(apiKey, inboxId)
+  if (typeof configOrApiKey === 'string') {
+    return getInboxEmailMailSlurp(configOrApiKey, inboxId ?? '');
+  }
+  if (configOrApiKey.provider === 'mailslurp' && configOrApiKey.apiKey && configOrApiKey.inboxId) {
+    return getInboxEmailMailSlurp(configOrApiKey.apiKey, configOrApiKey.inboxId);
+  }
+  if (configOrApiKey.provider === 'imap') {
+    return configOrApiKey.imapUser ?? null;
+  }
+  return null;
+}

package/src/engine-router.ts

@@ -0,0 +1,141 @@
+import type { DomainProfile } from './profiles/domain-profile.js';
+
+export type CrawlEngineName =
+  | 'http'
+  | 'browser'
+  | 'browser-stealth'
+  | 'stagehand'
+  | 'vision'
+  | 'recorded-session'
+  | 'native';
+
+export interface CrawlRouteDecisionInput {
+  appUrl: string;
+  appType?: string | null;
+  hasCredentials: boolean;
+  hasStorageState: boolean;
+  hasAuthToken: boolean;
+  hasSkyvern: boolean;
+  hasProxy: boolean;
+  profile: DomainProfile;
+  previousFailureReason?: string;
+}
+
+export interface CrawlRouteDecision {
+  engines: CrawlEngineName[];
+  reasons: string[];
+  confidence: number;
+}
+
+/**
+ * Per-domain rendering predictor — learns whether a domain requires a full browser
+ * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
+ *
+ * Strategy:
+ * 1. On first encounter of a domain, classify the HTTP response body.
+ * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
+ * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
+ */
+export class RenderingTypePredictor {
+  private cache = new Map<string, 'http' | 'browser'>();
+
+  /** Return cached decision if available, undefined if unknown. */
+  getCached(url: string): 'http' | 'browser' | undefined {
+    try {
+      return this.cache.get(new URL(url).hostname);
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * Analyze an HTTP response body to predict if JS rendering is required.
+   * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+   */
+  predict(url: string, htmlBody: string, statusCode: number): 'http' | 'browser' {
+    let decision: 'http' | 'browser' = 'http';
+    try {
+      if (statusCode >= 400) {
+        decision = 'browser'; // let browser handle error pages
+      } else {
+        const bodyLen = htmlBody.length;
+        // Strip tags to get raw text
+        const textOnly = htmlBody.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+        const wordCount = textOnly.split(/\s+/).filter(w => w.length > 2).length;
+
+        // SPA signals: very short text, presence of React/Vue/Angular root markers
+        const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody)
+          || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
+        const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
+        const tinyBody = bodyLen < 3000 && wordCount < 50;
+
+        if ((hasSpaRoot && wordCount < 100) || (tinyBody && !hasNoscriptFallback)) {
+          decision = 'browser';
+        }
+      }
+    } catch { /* default http */ }
+
+    try {
+      this.cache.set(new URL(url).hostname, decision);
+    } catch { /* ignore */ }
+    return decision;
+  }
+
+  /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+  set(url: string, engine: 'http' | 'browser'): void {
+    try { this.cache.set(new URL(url).hostname, engine); } catch { /* ignore */ }
+  }
+
+  stats(): { cached: number; browserDomains: string[]; httpDomains: string[] } {
+    const browserDomains: string[] = [];
+    const httpDomains: string[] = [];
+    for (const [domain, engine] of this.cache) {
+      if (engine === 'browser') browserDomains.push(domain);
+      else httpDomains.push(domain);
+    }
+    return { cached: this.cache.size, browserDomains, httpDomains };
+  }
+}
+
+/** Singleton predictor — shared across crawl jobs in the same process. */
+export const renderingPredictor = new RenderingTypePredictor();
+
+export function chooseCrawlEngines(input: CrawlRouteDecisionInput): CrawlRouteDecision {
+  const engines: CrawlEngineName[] = [];
+  const reasons: string[] = [];
+
+  const appType = input.appType?.toLowerCase();
+  if (appType === 'android' || appType === 'ios' || appType === 'windows' || appType === 'macos') {
+    return { engines: ['native'], reasons: [`appType=${input.appType}`], confidence: 0.95 };
+  }
+
+  if (input.hasStorageState || input.hasAuthToken) {
+    engines.push('recorded-session');
+    reasons.push('session/token auth available');
+  }
+
+  engines.push(input.hasProxy ? 'browser-stealth' : 'browser');
+  reasons.push(input.hasProxy ? 'proxy configured; prefer stealth browser path' : 'default browser path');
+
+  if (input.hasCredentials || input.profile.industry !== 'unknown') {
+    engines.push('stagehand');
+    reasons.push(input.hasCredentials ? 'credentials available for AI login/navigation' : `domain profile=${input.profile.industry}`);
+  }
+
+  const hardUi = input.profile.industry === 'legacy-enterprise' || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? '');
+  if (input.hasSkyvern && hardUi) {
+    engines.push('vision');
+    reasons.push('hard UI or previous block detected; vision fallback enabled');
+  }
+
+  if (engines.length === 1) {
+    engines.unshift('http');
+    reasons.push('HTTP/static extraction can be attempted before browser escalation');
+  }
+
+  return {
+    engines: Array.from(new Set(engines)),
+    reasons,
+    confidence: input.profile.confidence,
+  };
+}