@detiq/crawler 0.1.3

package/src/crawler.ts

@@ -0,0 +1,4305 @@
+/**
+ * ZeTa-AI Crawler — Stagehand hybrid architecture
+ *
+ * Three-layer stack:
+ *   Layer 1 — Playwright (deterministic): navigation, screenshots, DOM fallbacks
+ *   Layer 2 — Stagehand act/extract (AI): login, element understanding, smart link discovery
+ *   Layer 3 — Skyvern (vision): fallback for CAPTCHAs, legacy apps, visual-only flows
+ *
+ * enableCaching: true → repeat crawls of the same site cost ~zero AI tokens after first run.
+ */
+
+import { Stagehand } from '@browserbasehq/stagehand';
+// Option 2: rebrowser-playwright-core patches the CDP Runtime.enable leak at protocol level —
+// the deepest possible stealth, defeats detection methods that JS init scripts cannot touch.
+// @ts-ignore
+import { chromium } from 'rebrowser-playwright-core';
+import { chromium as stealthChromium } from 'playwright-extra';
+// @ts-ignore
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+// Option 3: Camoufox — Firefox-based browser with built-in fingerprint randomisation.
+// Loaded dynamically (not at startup) to avoid ESM/CJS conflict when API imports crawler.
+import { spawn, execSync, type ChildProcess } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+
+stealthChromium.use(StealthPlugin());
+import { z } from 'zod';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { AppBrain, recordLLMCall } from '@detiq/app-brain';
+import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
+import { config } from '@detiq/core';
+import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
+import type { Credentials } from './auth.js';
+import { skyvernFallback, skyvernNavigate, type SkyvernConfig } from './skyvern.js';
+import { loginWithStagehand } from './ai-login.js';
+import { readScreen } from './screen-reader.js';
+import { probeAllScreens } from './interactive-probe.js';
+import { computeElementHash } from './element-hash.js';
+import { logger } from './logger.js';
+import { solveCaptcha } from './captcha-solver.js';
+import { waitForEmailOTP } from './email-otp.js';
+import { getSsoToken } from './sso-token.js';
+import { discoverSitemapUrls, sameSite } from './sitemap.js';
+import { extractMarkdown } from './markdown-extract.js';
+import { CrawlFrontier, type TraversalStrategy } from './frontier.js';
+import { RobotsCache } from './robots-txt.js';
+import { classifyDomainProfile } from './profiles/domain-profile.js';
+import { chooseCrawlEngines } from './engine-router.js';
+import { assessActionSafety } from './policies/crawl-safety.js';
+import { attachNetworkObserver, type ApiEndpointObservationInput } from './network-observer.js';
+import { scrubPii } from './pii-scrubber.js';
+import { computeContentHash } from './incremental-diff.js';
+import { isPdfUrl, extractPdfFromUrl } from './pdf-extractor.js';
+import { attachWebSocketDetector } from './websocket-detector.js';
+import { ContentDeduplicator } from './content-dedup.js';
+import { guessGraphQLEndpoints, introspectGraphQL } from './graphql-detector.js';
+import { acceptCookieConsent } from './cookie-consent.js';
+import { detectJsRoutes } from './js-router-detector.js';
+import { runGapAnalysis } from './gap-pass.js';
+import { buildCrawlArtifactEnvelope, buildFitMarkdown, sha256, summarizeStructuredElements } from './extractors/artifact-envelope.js';
+import { pickFreeProxy, TieredProxyManager } from './proxy.js';
+import type { ProxyTierConfig } from './proxy.js';
+import { infiniteScrollToBottom, extractIframeContent, pierceShadowDom, injectFingerprintOverrides } from './stealth/human-behavior.js';
+import { checkSeo } from './seo-checker.js';
+import { auditColorContrast } from './color-contrast.js';
+import { detectFeatureFlags } from './feature-flag-detector.js';
+import { detectLanguage } from './language-detector.js';
+import { detectForms } from './form-detector.js';
+import { RedirectRegistry } from './redirect-tracker.js';
+import { DeadLinkTracker } from './dead-link-detector.js';
+import { screensToSitemapUrls, writeSitemap } from './sitemap-writer.js';
+import { generateOpenApiSpec, specToYaml } from './openapi-generator.js';
+import { WebhookNotifier } from './webhook-notifier.js';
+import { generatePlaywrightTest } from './playwright-recorder.js';
+
+function getGitInfo(): { branch: string; commit: string } {
+  try {
+    const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf8', timeout: 3000 }).trim();
+    const commit = execSync('git rev-parse HEAD', { encoding: 'utf8', timeout: 3000 }).trim().slice(0, 12);
+    return { branch: branch || 'main', commit };
+  } catch {
+    return { branch: 'main', commit: '' };
+  }
+}
+
+interface PerformanceMetric {
+  url: string;
+  screenName: string;
+  lcp?: number;
+  fcp?: number;
+  ttfb?: number;
+  cls?: number;
+}
+
+const ALLOWED_NUMERIC_PATHS = new Set(['/2fa', '/404', '/500', '/403', '/401']);
+const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
+const NUMERIC_ID_PATH_SEGMENT_RE = /^\d[\d-]*$/;
+const COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE = /^\d+-\d+$/;
+
+function normalizePathForNumericGuard(pathname: string): string {
+  const normalized = pathname.replace(/\/+$/, '') || '/';
+  return normalized;
+}
+
+function isGeneratedPathSegment(segment: string): boolean {
+  return UUID_PATH_SEGMENT_RE.test(segment)
+    || CUID_PATH_SEGMENT_RE.test(segment)
+    || COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE.test(segment)
+    || NUMERIC_ID_PATH_SEGMENT_RE.test(segment);
+}
+
+function isGeneratedDynamicPath(pathname: string): boolean {
+  const normalized = normalizePathForNumericGuard(pathname);
+  if (ALLOWED_NUMERIC_PATHS.has(normalized)) return false;
+
+  // Parent-route agnostic: any generated ID-like segment can create hundreds of
+  // duplicate detail/fallback screens under different sections of the app.
+  return normalized.split('/').filter(Boolean).some(isGeneratedPathSegment);
+}
+
+function isUsefulNavigationUrl(href: string, siteOrigin: string): boolean {
+  try {
+    const u = new URL(href);
+    if (!sameSite(u.toString(), siteOrigin)) return false;
+    if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
+    if (isGeneratedDynamicPath(u.pathname)) return false;
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+type AuthFlowType = 'public' | 'login' | 'register' | 'password_reset' | 'verification' | 'invite' | 'session_recovery';
+type AuthDiscoverySource = 'entry' | 'seed_path' | 'page_link' | 'auth_click';
+type AuthDiscoveryProgress = (pct: number, step: string) => void | Promise<void>;
+type AuthDiscoveryOptions = {
+  appUrl: string;
+  tenantId: string;
+  projectId: string;
+  jobId?: string;
+  screenshotDir?: string;
+};
+
+const AUTH_DISCOVERY_MAX_SCREENS = 36;
+const AUTH_DISCOVERY_MAX_QUEUE = 48;
+const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = 6;
+const AUTH_DISCOVERY_PATHS = [
+  '/',
+  '/login', '/log-in', '/signin', '/sign-in', '/auth', '/auth/login', '/account/login', '/user/login', '/app/login',
+  '/register', '/signup', '/sign-up', '/create-account', '/join',
+  '/forgot-password', '/forgot', '/password/forgot', '/reset-password', '/password/reset',
+  '/verify-email', '/invite',
+  '/pricing', '/contact', '/about', '/help', '/docs',
+];
+const AUTH_PATH_RE = /\/(login|log-in|signin|sign-in|auth|oauth|sso|account\/login|user\/login|app\/login|register|signup|sign-up|create-account|forgot|reset|password|verify-email|invite)(\/|$|\?)/i;
+const PUBLIC_PATH_RE = /\/($|pricing|contact|about|help|docs|features|security|privacy|terms)(\/|$|\?)/i;
+
+function compactSlug(input: string, fallback = 'state'): string {
+  const slug = input.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 48);
+  return slug || fallback;
+}
+
+function titleFromUrl(url: string, fallback = 'Page'): string {
+  try {
+    const u = new URL(url);
+    const parts = u.pathname.split('/').filter(Boolean);
+    if (parts.length === 0) return 'Home';
+    return parts
+      .slice(-2)
+      .join(' ')
+      .replace(/[-_]+/g, ' ')
+      .replace(/\b\w/g, (m) => m.toUpperCase());
+  } catch {
+    return fallback;
+  }
+}
+
+function flowTypeForUrlOrText(value: string): AuthFlowType {
+  const v = value.toLowerCase();
+  if (/(forgot|reset|password)/.test(v)) return 'password_reset';
+  if (/(register|signup|sign up|create account|join)/.test(v)) return 'register';
+  if (/(verify|verification|otp|mfa|2fa|invite)/.test(v)) return v.includes('invite') ? 'invite' : 'verification';
+  if (/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)/.test(v)) return 'session_recovery';
+  if (/(login|log in|signin|sign in|auth|sso|oauth)/.test(v)) return 'login';
+  return 'public';
+}
+
+function shouldSeedAuthDiscoveryUrl(url: string, origin: string): boolean {
+  try {
+    const u = new URL(url);
+    if (!sameSite(u.toString(), origin)) return false;
+    if (/\/(logout|logged-out|signout|sign-out|delete|remove|destroy)(\/|$|\?)/i.test(u.pathname)) return false;
+    return AUTH_PATH_RE.test(u.pathname) || PUBLIC_PATH_RE.test(u.pathname);
+  } catch {
+    return false;
+  }
+}
+
+function normalizeAuthUrl(url: string): string {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    return `${u.origin}${u.pathname.replace(/\/$/, '') || '/'}${u.search}`;
+  } catch {
+    return url;
+  }
+}
+
+async function capturePageHtml(page: any): Promise<string | null> {
+  try {
+    if (typeof page.content === 'function') {
+      const html = await page.content();
+      if (typeof html === 'string' && html.trim()) return html;
+    }
+  } catch (err: any) {
+    logger.warn({ err: String(err?.message ?? err) }, '[crawler] page.content() failed; trying DOM evaluate fallback');
+  }
+
+  try {
+    const html = await page.evaluate(() => {
+      const type = document.doctype;
+      const doctype = type
+        ? '<!DOCTYPE ' + type.name +
+          (type.publicId ? ' PUBLIC "' + type.publicId + '"' : '') +
+          (type.systemId ? ' "' + type.systemId + '"' : '') +
+          '>'
+        : '<!DOCTYPE html>';
+      return doctype + '\n' + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? '');
+    });
+    return typeof html === 'string' && html.trim() ? html : null;
+  } catch (err: any) {
+    logger.warn({ err: String(err?.message ?? err) }, '[crawler] DOM snapshot capture failed');
+    return null;
+  }
+}
+
+/** Launch Chrome on a random CDP port and return its WebSocket debugger URL. */
+async function launchChromeCDP(
+  execPath: string,
+  extraArgs: string[],
+  proxy?: { server: string; username?: string; password?: string },
+): Promise<{ wsUrl: string; proc: any }> {
+  // Use a random port in 9200-9900 range to avoid clashes between concurrent crawls.
+  const port = 9200 + Math.floor(Math.random() * 700);
+
+  const browserServer = await stealthChromium.launchServer({
+    headless: true,
+    executablePath: execPath,
+    // Use Playwright's native proxy option — it handles 407 auth challenges automatically.
+    // The --proxy-server Chrome flag does NOT support credentials with special chars (commas).
+    ...(proxy ? { proxy } : {}),
+    args: [
+      ...extraArgs,
+      `--remote-debugging-port=${port}`,
+      '--remote-allow-origins=*',
+      '--no-first-run',
+      '--no-default-browser-check',
+    ],
+  });
+
+  // Poll until Chrome's debug port is ready (up to 10 s).
+  const deadline = Date.now() + 10_000;
+  while (Date.now() < deadline) {
+    await new Promise((r) => setTimeout(r, 300));
+    try {
+      const res = await fetch(`http://127.0.0.1:${port}/json/version`);
+      if (res.ok) {
+        const { webSocketDebuggerUrl } = await res.json() as { webSocketDebuggerUrl: string };
+        return { wsUrl: webSocketDebuggerUrl, proc: browserServer };
+      }
+    } catch { /* not ready */ }
+  }
+  await browserServer.close();
+  throw new Error(`Chrome failed to open CDP port ${port} within 10 s`);
+}
+
+/**
+ * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser
+ * and returns same-site `<a href>` links. Complements sitemap discovery for SPA/app sites that
+ * have no sitemap.xml. Best-effort — returns [] on any failure (bot wall, timeout, etc.).
+ * chromium.launch({ proxy }) handles proxy 407 auth itself (same client), so no CDP handler needed.
+ */
+export async function discoverLinksViaBrowser(
+  appUrl: string,
+  opts?: { proxyUrl?: string; maxLinks?: number },
+): Promise<string[]> {
+  let proxy: { server: string; username?: string; password?: string } | undefined;
+  const purl = opts?.proxyUrl;
+  if (purl && purl !== 'auto') {
+    const m = purl.match(/^(https?|socks5?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (m) {
+      const [, scheme, user, pass, hostPort] = m;
+      proxy = user
+        ? { server: `${scheme}://${hostPort}`, username: decodeURIComponent(user), password: decodeURIComponent(pass ?? '') }
+        : { server: `${scheme}://${hostPort}` };
+    } else {
+      proxy = { server: purl };
+    }
+  }
+  let executablePath: string | undefined;
+  try { executablePath = chromium.executablePath(); } catch { /* fall back to CHROME_PATH */ }
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let browser: any = null;
+  try {
+    browser = await chromium.launch({
+      headless: true,
+      ...(executablePath ? { executablePath } : {}),
+      ...(proxy ? { proxy } : {}),
+      args: ['--no-sandbox', '--disable-dev-shm-usage'],
+    });
+    const page = await browser.newPage();
+    await page.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => { });
+    await page.waitForTimeout(2_500); // let SPA / Web Components hydrate before scraping links
+    const hrefs: string[] = await page.evaluate((origin: string) => {
+      const found = new Set<string>();
+      const walk = (root: Document | ShadowRoot | Element) => {
+        (root as any).querySelectorAll('a[href]').forEach((a: any) => {
+          const h: string = a.href || a.getAttribute('href') || '';
+          if (h && (h.startsWith(origin) || h.startsWith('/'))) found.add(h);
+        });
+        (root as any).querySelectorAll('*').forEach((el: any) => {
+          if (el.shadowRoot) walk(el.shadowRoot);
+        });
+      };
+      walk(document);
+      return [...found];
+    }, new URL(appUrl).origin).catch(() => [] as string[]);
+    return [...new Set(hrefs.filter((h) => sameSite(h, appUrl)))].slice(0, opts?.maxLinks ?? 500);
+  } catch {
+    return [];
+  } finally {
+    if (browser) await browser.close().catch(() => { });
+  }
+}
+
+/**
+ * Run a Lighthouse audit against a URL and return category scores (0-100 integers).
+ * Requires ENABLE_LIGHTHOUSE=true env var — disabled by default to avoid slowing crawls.
+ * Returns null on any error or when disabled.
+ */
+async function runLighthouse(url: string): Promise<{
+  perf: number; a11y: number; seo: number; bp: number; report: object;
+} | null> {
+  if (!process.env.ENABLE_LIGHTHOUSE) return null;
+  try {
+    const { default: lighthouse } = await import('lighthouse');
+    const chromeLauncher = await import('chrome-launcher');
+    const chrome = await chromeLauncher.launch({
+      chromeFlags: ['--headless', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'],
+    });
+    try {
+      const result = await lighthouse(url, {
+        port: chrome.port,
+        output: 'json',
+        logLevel: 'error',
+        onlyCategories: ['performance', 'accessibility', 'seo', 'best-practices'],
+      } as any);
+      if (!result?.lhr?.categories) return null;
+      const cats = result.lhr.categories as Record<string, { score: number | null }>;
+      const score = (key: string) => Math.round((cats[key]?.score ?? 0) * 100);
+      return {
+        perf: score('performance'),
+        a11y: score('accessibility'),
+        seo: score('seo'),
+        bp: score('best-practices'),
+        report: result.lhr as object,
+      };
+    } finally {
+      await chrome.kill();
+    }
+  } catch (err) {
+    console.warn('[crawler] Lighthouse run failed (non-fatal):', err);
+    return null;
+  }
+}
+
+const ElementSchema = z.object({
+  elements: z.array(z.object({
+    meaning: z.string(),
+    role: z.string(),
+    expectedData: z.string().optional(),
+  })),
+});
+
+const LinksSchema = z.object({
+  links: z.array(z.string()),
+});
+
+const TwoFASchema = z.object({
+  has2FA: z.boolean(),
+  fieldLabel: z.string().optional(),
+});
+
+export async function crawlProject(opts: {
+  tenantId: string;
+  projectId: string;
+  appUrl: string;
+  credentials?: Credentials;
+  /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
+  storageState?: string;
+  screenshotDir?: string;
+  /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
+  discoveryMode?: boolean;
+  maxScreens?: number;
+  /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+  startUrls?: string[];
+  /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
+  selectiveUrls?: string[];
+  /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
+  interactive?: boolean;
+  /** Optional abort signal — abort() cancels the crawl between page navigations. */
+  abortSignal?: AbortSignal;
+  /** Platform job ID — stored on screens so we can diff which were found in each crawl. */
+  jobId?: string;
+  /** Device profiles to capture screenshots for after the main DESKTOP crawl.
+   *  Values: 'MOBILE_PORTRAIT' | 'TABLET'
+   *  The main crawl always runs as DESKTOP. Extra profiles trigger a screenshot-only pass.
+   */
+  deviceProfiles?: string[];
+  /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+  httpBasicUsername?: string;
+  httpBasicPassword?: string;
+  /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+  authToken?: string;
+  /** Header name for authToken (default: 'Authorization'). */
+  authTokenHeader?: string;
+  /** Prefix for authToken value (default: 'Bearer '). */
+  authTokenPrefix?: string;
+  /** CAPTCHA solving API key (2captcha or CapSolver). */
+  captchaSolverApiKey?: string;
+  /** CAPTCHA solving service provider (default: '2captcha'). */
+  captchaSolverProvider?: '2captcha' | 'capsolver';
+  /**
+   * Path to a Chrome user data directory for profile inheritance.
+   * When set, the crawler exports the existing session from this profile and injects it,
+   * so the agent is already logged in. Works for local/self-hosted deployments only.
+   * Ignored on cloud/Docker (headless env cannot open another Chrome instance safely).
+   */
+  chromeProfilePath?: string;
+  /** MailSlurp API key for email OTP auto-retrieval (GAP 5). */
+  mailslurpApiKey?: string;
+  /** MailSlurp inbox ID to poll for OTP emails (GAP 5). */
+  mailslurpInboxId?: string;
+  /** SSO provider for ROPC token flow — 'okta' | 'azure_ad' | 'generic_oidc' (GAP 18). */
+  ssoProvider?: string;
+  /** SSO domain or Azure tenant ID (GAP 18). */
+  ssoDomain?: string;
+  /** SSO OAuth2 client ID (GAP 18). */
+  ssoClientId?: string;
+  /** SSO OAuth2 client secret (GAP 18). */
+  ssoClientSecret?: string;
+  /** SSO scopes (space-separated, default: 'openid profile') (GAP 18). */
+  ssoScope?: string;
+  /** HTTP/HTTPS/SOCKS5 proxy URL — e.g. 'http://user:pass@proxy.example.com:8080' or 'socks5://proxy:1080'.
+   *  Useful for sites that block datacenter IPs (Akamai, Cloudflare Bot Manager). */
+  proxyUrl?: string;
+  /** Tiered proxy configuration for automatic per-domain escalation (free → datacenter → residential → premium). */
+  proxyTiers?: ProxyTierConfig[];
+  /** Use Camoufox (Firefox) as a cookie bootstrapper before the main Chromium crawl.
+   *  Camoufox visits the URL first to solve Cloudflare JS challenges and extract session
+   *  cookies, which are then injected into the Chromium crawl as storageState.
+   *  Enable via Project Settings → Crawl Config → useFirefox: true. */
+  useFirefox?: boolean;
+  /** Natural language login instructions for Stagehand AI login. E.g. "Click Sign In in the nav, fill email, click Continue". */
+  loginInstructions?: string;
+  /** Skyvern vision fallback config from workspace settings (overrides SKYVERN_API_URL/KEY env vars). */
+  skyvernConfig?: SkyvernConfig;
+  /**
+   * Max screens to crawl per segment (default 50).
+   * After this limit the crawl pauses and remaining URLs are saved so the
+   * next job can continue. Total crawl is unlimited — just runs in multiple jobs.
+   */
+  segmentSize?: number;
+  /** URLs still queued from a previous segment — injected by the job-queue auto-continuation. */
+  resumeUrls?: string[];
+  /** Segment number (1-based) — used for progress display only. */
+  segmentIndex?: number;
+  /** Seed the crawl queue from the project's sitemap.xml, in addition to link-following. */
+  useSitemap?: boolean;
+  /** Explicit sitemap URL override — defaults to `${origin}/sitemap.xml` when useSitemap is set. */
+  sitemapUrl?: string;
+  /** Crawl queue traversal order. Default BFS — unchanged behavior from before this option existed. */
+  traversalStrategy?: TraversalStrategy;
+  /** BEST_FIRST only: URLs containing any of these (case-insensitive) are prioritized. */
+  bestFirstKeywords?: string[];
+  /** When true, skip URLs disallowed by the site's robots.txt. Default false. */
+  respectRobotsTxt?: boolean;
+  /** When true, dynamically slow crawl speed when server response latency is high (Scrapy-style AutoThrottle). Default false. */
+  autoThrottle?: boolean;
+  /** Regex pattern — only crawl URLs whose path+query match this pattern. */
+  urlFilter?: string;
+  /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
+  skipUrls?: string[];
+  /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
+  knownDomHashes?: string[];
+  /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
+  onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
+  /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
+  onUrlsDiscovered?: (urls: string[]) => Promise<void>;
+}, onProgress?: (pct: number, step: string) => Promise<void>) {
+  if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
+
+  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === 'production';
+
+  // Use tenant LLM config if available; fall back to env vars (Anthropic only for Stagehand).
+  // Stagehand v3 requires model in "provider/model" format (e.g. "anthropic/claude-sonnet-4-6").
+  const toStagehandModel = (model: string, provider: string) =>
+    model.includes('/') ? model : `${provider}/${model}`;
+
+  let stagehandModel = toStagehandModel(config.llmModel, 'openai');
+  let stagehandApiKey = '';
+  let configuredProvider = '';
+  try {
+    // getTaskAgentLLMConfig: tenant config → platform admin config → null (proper fallback chain)
+    const crawlLLM = await getTaskAgentLLMConfig(opts.tenantId, 'crawl');
+    if (crawlLLM) {
+      configuredProvider = crawlLLM.provider;
+      if (crawlLLM.apiKey && crawlLLM.model) {
+        if (crawlLLM.provider === 'anthropic' || crawlLLM.provider === 'openai') {
+          stagehandModel = toStagehandModel(crawlLLM.model, crawlLLM.provider);
+          stagehandApiKey = crawlLLM.apiKey;
+        } else if (crawlLLM.provider === 'openrouter') {
+          const model = crawlLLM.model.includes('/') ? crawlLLM.model : `openai/${crawlLLM.model}`;
+          stagehandModel = model;
+          stagehandApiKey = crawlLLM.apiKey;
+        }
+      }
+    }
+  } catch { /* ignore DB errors, fall through to env var below */ }
+  // Env var fallback — if platform/tenant config lookup failed or returned no key
+  if (!stagehandApiKey) {
+    // Honor STAGEHAND_MODEL env var to determine preferred provider.
+    // Stagehand v3 requires "provider/model" format — mismatched pair silently fails.
+    const envModel = process.env.STAGEHAND_MODEL ?? '';
+    const preferAnthropic = envModel.includes('claude') || (!envModel && !process.env.OPENAI_API_KEY);
+    if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
+      stagehandApiKey = process.env.ANTHROPIC_API_KEY;
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+    } else if (process.env.OPENAI_API_KEY) {
+      stagehandApiKey = process.env.OPENAI_API_KEY;
+      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || 'gpt-4o'}`;
+    } else if (process.env.ANTHROPIC_API_KEY) {
+      stagehandApiKey = process.env.ANTHROPIC_API_KEY;
+      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || 'claude-sonnet-4-6'}`;
+    }
+  }
+
+  const elementLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, 'elementExtraction').catch(() => null);
+  const linkLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, 'linkDiscovery').catch(() => null);
+  const projectForProfile = await AppBrain.getProjectMap(opts.tenantId, opts.projectId).catch(() => null);
+  const profileProjectName = projectForProfile?.name ?? null;
+  const profileAppType = projectForProfile?.appType ?? null;
+  const domainProfile = classifyDomainProfile({
+    appUrl: opts.appUrl,
+    projectName: profileProjectName,
+    appType: profileAppType,
+    seedText: [
+      ...(opts.startUrls ?? []),
+      opts.loginInstructions ?? '',
+      ...(opts.bestFirstKeywords ?? []),
+    ].join(' '),
+  });
+  const routeDecision = chooseCrawlEngines({
+    appUrl: opts.appUrl,
+    appType: profileAppType,
+    hasCredentials: !!opts.credentials,
+    hasStorageState: !!opts.storageState,
+    hasAuthToken: !!opts.authToken,
+    hasSkyvern: !!opts.skyvernConfig,
+    hasProxy: !!opts.proxyUrl,
+    profile: domainProfile,
+  });
+
+  AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+    jobId: opts.jobId,
+    type: 'CRAWL_PROFILE',
+    payload: { profile: domainProfile, routeDecision },
+  }).catch((err: any) => console.warn('[crawler] profile evidence save failed:', err?.message ?? err));
+
+  if (!stagehandApiKey) {
+    const unsupported = configuredProvider && !['anthropic', 'openai', 'openrouter'].includes(configuredProvider);
+    throw new Error(
+      unsupported
+        ? `Browser crawling requires Anthropic or OpenAI. Your primary provider "${configuredProvider}" is not supported for crawling. Add a separate Crawl API Key in Workspace → Settings → AI Provider.`
+        : 'No AI provider configured. Go to Workspace → Settings → AI Provider to set up Anthropic or OpenAI for browser crawling.'
+    );
+  }
+
+  // Anti-detect args applied on all platforms.
+  // '--disable-blink-features=AutomationControlled' removes navigator.webdriver at the
+  // Chromium level — more reliable than JS overrides alone.
+  // '--disable-http2' defeats Akamai/Cloudflare JA3/JA4 TLS fingerprinting on datacenter IPs.
+  const stealthArgs = [
+    '--disable-blink-features=AutomationControlled',
+    '--disable-http2',
+    '--lang=en-US,en',
+    '--window-size=1366,768',
+    '--disable-features=IsolateOrigins,site-per-process',
+  ];
+
+  const chromeArgs = isDocker
+    ? [
+      '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu',
+      ...stealthArgs,
+    ]
+    : [...stealthArgs];
+  // Resolve proxy: tiered manager > 'auto' free list > explicit URL
+  let resolvedProxyUrl = opts.proxyUrl;
+  let tieredProxyManager: TieredProxyManager | undefined;
+  if (opts.proxyTiers?.length) {
+    tieredProxyManager = new TieredProxyManager(opts.proxyTiers);
+    const domain = (() => { try { return new URL(opts.appUrl).hostname; } catch { return opts.appUrl; } })();
+    resolvedProxyUrl = tieredProxyManager.selectProxy(domain) ?? resolvedProxyUrl;
+    logger.info({ domain, proxyUrl: resolvedProxyUrl }, '[crawler] tiered proxy selected');
+  } else if (resolvedProxyUrl === 'auto') {
+    logger.info('[crawler] proxyUrl=auto: selecting free proxy from public list...');
+    resolvedProxyUrl = (await pickFreeProxy()) ?? undefined;
+    if (resolvedProxyUrl) {
+      logger.info({ proxyUrl: resolvedProxyUrl }, '[crawler] auto-proxy selected');
+    } else {
+      logger.warn('[crawler] auto-proxy: no working proxy found, proceeding without proxy');
+    }
+  }
+  // Parse proxy URL into Playwright's native proxy format.
+  // We do NOT use --proxy-server Chrome arg because:
+  //   1. Chrome ignores credentials in --proxy-server URLs (no auth support via flag)
+  //   2. Commas in username (DataImpulse country codes) break Chrome's proxy-list parser
+  // Playwright's launchServer({ proxy }) handles 407 auth challenges correctly.
+  let playwrightProxy: { server: string; username?: string; password?: string } | undefined;
+  if (resolvedProxyUrl) {
+    try {
+      // Manual parse: new URL() chokes on commas in the username part
+      const proxyMatch = resolvedProxyUrl.match(/^(https?|socks5?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+      if (proxyMatch) {
+        const [, scheme, rawUser, rawPass, hostPort] = proxyMatch;
+        const server = `${scheme}://${hostPort}`;
+        const username = rawUser ? decodeURIComponent(rawUser) : undefined;
+        const password = rawPass ? decodeURIComponent(rawPass) : undefined;
+        playwrightProxy = username ? { server, username, password } : { server };
+        logger.info({ server, hasAuth: !!username }, '[crawler] proxy configured (Playwright native)');
+      } else {
+        playwrightProxy = { server: resolvedProxyUrl };
+      }
+    } catch {
+      playwrightProxy = { server: resolvedProxyUrl };
+    }
+  }
+
+  // Stagehand 3.x uses chrome-launcher under the hood, which fails in Docker
+  // because it doesn't pass --no-sandbox and uses a different spawn environment.
+  // Fix: spawn Chrome ourselves with the right flags, get its CDP WS URL,
+  // then connect Stagehand to the already-running browser via cdpUrl.
+  let executablePath: string | undefined;
+  try { executablePath = chromium.executablePath(); } catch { /* use CHROME_PATH */ }
+
+  if (!executablePath) throw new Error('No Chrome binary found. Set CHROME_PATH or install Playwright browsers.');
+
+  // GAP 3: Chrome Profile Inheritance — local/self-hosted deployments only.
+  // Launch a temporary persistent context to export the storageState from the user's Chrome profile,
+  // then inject it so the normal session-restore path handles auth (no AI login needed).
+  let effectiveStorageState = opts.storageState;
+  if (opts.chromeProfilePath && !isDocker) {
+    logger.info({ chromeProfilePath: opts.chromeProfilePath }, '[crawler] Exporting session from Chrome profile');
+    try {
+      const tempCtx = await chromium.launchPersistentContext(opts.chromeProfilePath, {
+        headless: true,
+        args: [...chromeArgs, '--no-first-run', '--no-default-browser-check'],
+        executablePath,
+      });
+      const exportedState = await tempCtx.storageState();
+      await tempCtx.close().catch(() => { });
+      if ((exportedState.cookies?.length ?? 0) > 0 || (exportedState.origins?.length ?? 0) > 0) {
+        effectiveStorageState = JSON.stringify(exportedState);
+        logger.info({ cookies: exportedState.cookies?.length ?? 0, origins: exportedState.origins?.length ?? 0 }, '[crawler] Chrome profile session exported and injected');
+      } else {
+        logger.warn('[crawler] Chrome profile had no session data — continuing without profile auth');
+      }
+    } catch (profileErr) {
+      logger.warn({ err: profileErr }, '[crawler] Could not load Chrome profile — continuing without profile auth');
+    }
+  } else if (opts.chromeProfilePath && isDocker) {
+    logger.warn('[crawler] chromeProfilePath is set but will be ignored in Docker/cloud environments');
+  }
+
+  // ── Option 3: Camoufox bootstrap ─────────────────────────────────────────────
+  // Firefox with built-in fingerprint randomisation visits the target first to solve
+  // any Cloudflare JS challenge, then we extract the cf_clearance cookie and inject it
+  // into the Chromium storageState so Chrome inherits the cleared session.
+  if (opts.useFirefox) {
+    logger.info('[crawler] useFirefox: launching Camoufox to bootstrap Cloudflare cookies...');
+    try {
+      // Dynamic import prevents ESM/CJS conflict at API startup when this module is imported
+      const { NewBrowser: camoufoxNewBrowser } = await import('camoufox');
+      const pw = require('playwright');
+      const foxBrowser = await camoufoxNewBrowser(pw, true);
+      const foxPage = await foxBrowser.newPage();
+      await foxPage.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
+      // Give Cloudflare JS challenge up to 8s to complete and set cf_clearance cookie
+      await foxPage.waitForTimeout(8_000);
+      const foxState = await foxPage.context().storageState();
+      await foxBrowser.close().catch(() => { });
+      if ((foxState.cookies?.length ?? 0) > 0) {
+        // Merge Camoufox cookies into any existing storageState
+        if (effectiveStorageState) {
+          try {
+            const existing = JSON.parse(effectiveStorageState);
+            const merged = { ...existing, cookies: [...(existing.cookies ?? []), ...foxState.cookies] };
+            effectiveStorageState = JSON.stringify(merged);
+          } catch {
+            effectiveStorageState = JSON.stringify(foxState);
+          }
+        } else {
+          effectiveStorageState = JSON.stringify(foxState);
+        }
+        logger.info({ cookies: foxState.cookies.length }, '[crawler] Camoufox bootstrap complete — cf_clearance injected into Chromium');
+      } else {
+        logger.warn('[crawler] Camoufox returned no cookies — continuing without bootstrap');
+      }
+    } catch (foxErr) {
+      logger.warn({ err: foxErr }, '[crawler] Camoufox bootstrap failed — continuing without it');
+    }
+  }
+
+  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy);
+
+  // Accumulate token usage from Stagehand's internal LLM calls (extract/act/observe).
+  // Stagehand doesn't expose per-call usage externally; we parse it from its logger.
+  let stagehandPromptTokens = 0;
+  let stagehandCompletionTokens = 0;
+  const stagehandLogger = (line: { category?: string; message: string; auxiliary?: Record<string, { value: string; type: string }> }) => {
+    if (line.message === 'response' && line.auxiliary?.response?.value) {
+      try {
+        const resp = JSON.parse(line.auxiliary.response.value);
+        if (resp?.usage) {
+          stagehandPromptTokens += resp.usage.prompt_tokens ?? 0;
+          stagehandCompletionTokens += resp.usage.completion_tokens ?? 0;
+        }
+      } catch { /* ignore parse errors */ }
+    }
+  };
+
+  const stagehand = new Stagehand({
+    env: 'LOCAL',
+    model: { modelName: stagehandModel as any, apiKey: stagehandApiKey },
+    serverCache: true,
+    verbose: 1,
+    logger: stagehandLogger,
+    localBrowserLaunchOptions: { cdpUrl, headless: true },
+    domSettleTimeout: 2000,
+  });
+
+  await stagehand.init();
+  await onProgress?.(5, 'Browser launched');
+  // In Stagehand v3, stagehand.page is undefined when connected via CDP.
+  // Use resolvePage() to get the active V3Page (which is a full Playwright proxy).
+  const page = await (stagehand as any).resolvePage();
+  // Cap all Playwright operations so one stuck page can't freeze the crawl.
+  try { page.setDefaultTimeout(45_000); } catch { /* proxy may not expose this */ }
+  try { page.setDefaultNavigationTimeout(30_000); } catch { /* proxy may not expose this */ }
+  // Stagehand V3Page proxy only supports the 'console' event (page.on throws for others).
+  // rawPage will be wired to a real Playwright Page via the CDP connection established below.
+  // Declared as `let` so the network-observer block can populate it.
+  let rawPage: any = null; // populated after playwrightBrowser connects
+
+  // ── Proxy auth ───────────────────────────────────────────────────────────────
+  // Proxy 407 auth is answered via a CDP Fetch.continueWithAuth handler installed on
+  // the connectOverCDP page below (see "proxy auth via CDP Fetch handler"). page.authenticate()
+  // is a Puppeteer API — it does NOT exist on Playwright's connectOverCDP page and throws,
+  // leaving 407 challenges unanswered (net::ERR_INVALID_AUTH_CREDENTIALS).
+
+  // ── Stealth: defeat bot detection (Cloudflare, Akamai, DataDome) ────────────
+  // addInitScript runs in every new document context BEFORE any page JS executes.
+  // This overrides navigator.webdriver before anti-bot scripts can read it.
+  await page.addInitScript(() => {
+    // Core automation flags
+    Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+
+    // Realistic plugin list (Cloudflare checks length > 0)
+    const fakePlugins = ['PDF Viewer', 'Chrome PDF Viewer', 'Chromium PDF Viewer', 'Microsoft Edge PDF Viewer', 'WebKit built-in PDF'];
+    Object.defineProperty(navigator, 'plugins', {
+      get: () => Object.assign(fakePlugins.map((name, i) => ({ name, filename: 'internal-pdf-viewer', description: '', length: 1, item: () => null, namedItem: () => null })), { item: (i: number) => null, namedItem: () => null, refresh: () => { } }),
+    });
+
+    // @ts-ignore
+    window.chrome = { runtime: {}, loadTimes: () => ({}), csi: () => ({}), app: {} };
+
+    // Cloudflare checks navigator.permissions.query — headless returns "denied", real = "prompt"
+    const origQuery = window.navigator.permissions?.query?.bind(navigator.permissions);
+    if (origQuery) {
+      // @ts-ignore
+      navigator.permissions.query = (params: any) =>
+        params?.name === 'notifications'
+          ? Promise.resolve({ state: 'prompt', onchange: null } as PermissionStatus)
+          : origQuery(params);
+    }
+
+    // Cloudflare checks Notification.permission — "denied" = bot signal
+    try { Object.defineProperty(Notification, 'permission', { get: () => 'default' }); } catch { }
+
+    // navigator.connection — absence signals headless
+    // @ts-ignore
+    if (!navigator.connection) Object.defineProperty(navigator, 'connection', { get: () => ({ rtt: 50, downlink: 10, effectiveType: '4g', saveData: false }) });
+
+    // Remove Playwright/CDP globals leaked into page context
+    // @ts-ignore
+    delete window.__playwright; delete (window as any).__pw_manual; delete (window as any).__PW_inspect;
+  });
+
+  // ── Web Vitals capture: LCP + CLS ────────────────────────────────────────────
+  // LCP/CLS entries are only delivered to a PerformanceObserver registered with
+  // buffered:true — performance.getEntriesByType(...) after the fact returns [].
+  // addInitScript re-runs on every new document (survives page.goto navigations),
+  // so the observers are always registered before the page paints. The crawl loop
+  // reads the stashed values off window.__zetaLCP / window.__zetaCLS after load.
+  await page.addInitScript(() => {
+    try {
+      // @ts-ignore — custom stash properties
+      window.__zetaLCP = null;
+      // @ts-ignore
+      window.__zetaCLS = 0;
+      new PerformanceObserver((list) => {
+        const entries = list.getEntries();
+        const last = entries[entries.length - 1] as any;
+        // @ts-ignore
+        if (last) window.__zetaLCP = last.startTime;
+      }).observe({ type: 'largest-contentful-paint', buffered: true });
+      new PerformanceObserver((list) => {
+        for (const entry of list.getEntries() as any[]) {
+          // Ignore shifts triggered by user input (crawler clicks/scrolls).
+          // @ts-ignore
+          if (!entry.hadRecentInput) window.__zetaCLS += entry.value;
+        }
+      }).observe({ type: 'layout-shift', buffered: true });
+    } catch { /* older engines without these entry types — leave stashed values at defaults */ }
+  });
+
+  // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
+  // CDP-connected contexts cannot use browser.newContext({ httpCredentials }),
+  // so we intercept via page.route() instead.
+  if (opts.httpBasicUsername && opts.httpBasicPassword) {
+    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
+    const appOrigin = new URL(opts.appUrl).origin;
+    await page.route('**/*', (route: any) => {
+      const reqUrl = route.request().url();
+      if (reqUrl.startsWith(appOrigin)) {
+        route.continue({
+          headers: { ...route.request().headers(), 'Authorization': `Basic ${encoded}` },
+        });
+      } else {
+        route.continue();
+      }
+    });
+  }
+
+  // ── API/Bearer token injection ────────────────────────────────────────────
+  // If an authToken is provided, inject it as an HTTP header on every request
+  // and store it in localStorage so client-side code can pick it up.
+  if (opts.authToken) {
+    const headerName = opts.authTokenHeader ?? 'Authorization';
+    const prefix = opts.authTokenPrefix ?? 'Bearer ';
+    await page.setExtraHTTPHeaders({ [headerName]: `${prefix}${opts.authToken}` });
+    await page.addInitScript((token: string) => {
+      localStorage.setItem('auth_token', token);
+      localStorage.setItem('access_token', token);
+      localStorage.setItem('token', token);
+    }, opts.authToken);
+  }
+
+  // ── Web Vitals capture: CLS via buffered PerformanceObserver ─────────────────
+  // CLS entries are only delivered to a PerformanceObserver; this init script
+  // re-runs on every new document. LCP is captured per-page in the crawl loop.
+  await page.addInitScript(() => {
+    try {
+      // @ts-ignore — custom stash property
+      window.__zetaCLS = 0;
+      new PerformanceObserver((list) => {
+        for (const entry of list.getEntries() as any[]) {
+          // Ignore shifts triggered by user input (crawler clicks/scrolls).
+          // @ts-ignore
+          if (!entry.hadRecentInput) window.__zetaCLS += entry.value;
+        }
+      }).observe({ type: 'layout-shift', buffered: true });
+    } catch { /* Older engines without layout-shift entry type — leave at 0 */ }
+  });
+
+  // Intercept SPA route changes (history.pushState / replaceState) to discover JS-router routes
+  // that have no <a href> elements — common in React Router, Next.js, Vue Router apps.
+  await page.addInitScript(() => {
+    (window as any).__zetaDiscoveredRoutes = (window as any).__zetaDiscoveredRoutes ?? [];
+    const push = history.pushState.bind(history);
+    const replace = history.replaceState.bind(history);
+    history.pushState = function (data: any, unused: string, url?: string | URL | null) {
+      const result = push(data, unused, url);
+      if (url && typeof url === 'string') (window as any).__zetaDiscoveredRoutes.push(url);
+      return result;
+    };
+    history.replaceState = function (data: any, unused: string, url?: string | URL | null) {
+      const result = replace(data, unused, url);
+      if (url && typeof url === 'string') (window as any).__zetaDiscoveredRoutes.push(url);
+      return result;
+    };
+  });
+
+  // Inject canvas/WebGL fingerprint overrides + hardware concurrency spoof
+  await injectFingerprintOverrides(page).catch(() => { });
+
+  // ── Network interception: capture REST/GraphQL API endpoints ────────────────
+  // Stagehand v3's page proxy only supports the 'console' event — to intercept
+  // network requests we connect a second Playwright client to the same CDP URL.
+  const capturedApiEndpoints: ApiEndpointObservationInput[] = [];
+  // P3a: URLs flagged for re-visit after async content detected (drained after main BFS)
+  const revisitQueue: string[] = [];
+  // P4: XHR-discovered page URL candidates (relative paths from JSON responses)
+  const xhrDiscoveredPaths = new Set<string>();
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let playwrightBrowser: any = null;
+  try {
+    playwrightBrowser = await chromium.connectOverCDP(cdpUrl);
+    const pwContext = playwrightBrowser.contexts()[0];
+    if (pwContext) {
+      // P4: pass XHR URL discovery callback — adds relative paths to xhrDiscoveredPaths
+      attachNetworkObserver(pwContext, capturedApiEndpoints, (relPath) => xhrDiscoveredPaths.add(relPath));
+      // OAuth/SSO popup handler — attach before login so we don't miss it
+      pwContext.on('page', (popup: any) => {
+        const popupUrl = popup.url();
+        logger.info({ url: popupUrl }, '[crawler] OAuth popup detected — waiting for close');
+        popup.waitForEvent('close', { timeout: 120_000 })
+          .then(() => logger.info('[crawler] OAuth popup closed — SSO flow complete'))
+          .catch(() => logger.warn('[crawler] OAuth popup timed out or errored'));
+      });
+    }
+    // Wire rawPage to the real Playwright page from this CDP connection.
+    // This gives us a true Playwright Page that supports all events (.on('response'), etc.)
+    // while `page` (Stagehand) is still used for AI actions (act/extract/observe).
+    const pwContext2 = playwrightBrowser?.contexts()?.[0];
+    if (pwContext2) {
+      const pwPages = pwContext2.pages();
+      rawPage = pwPages.find((p: any) => p.url() !== 'about:blank') ?? pwPages[0] ?? rawPage;
+    }
+
+    // ── Proxy auth via CDP Fetch handler ──────────────────────────────────────
+    // launchServer({ proxy }) applies the proxy at the Chrome flag level, but its
+    // credentials are only honored by Playwright's OWN launching client. Since we
+    // drive the browser through a separate connectOverCDP client, the proxy 407 goes
+    // unanswered (net::ERR_INVALID_AUTH_CREDENTIALS). Answer it via the CDP Fetch domain,
+    // then disable interception — Chrome caches the proxy credentials for the rest of the session.
+    if (playwrightProxy?.username && pwContext2 && rawPage && rawPage !== page) {
+      try {
+        const proxyCdp = await pwContext2.newCDPSession(rawPage);
+        await proxyCdp.send('Fetch.enable', { handleAuthRequests: true, patterns: [{ urlPattern: '*' }] });
+        proxyCdp.on('Fetch.requestPaused', (e: any) => {
+          proxyCdp.send('Fetch.continueRequest', { requestId: e.requestId }).catch(() => { });
+        });
+        let proxyAuthed = false;
+        proxyCdp.on('Fetch.authRequired', async (e: any) => {
+          const isProxy = e.authChallenge?.source === 'Proxy';
+          try {
+            await proxyCdp.send('Fetch.continueWithAuth', {
+              requestId: e.requestId,
+              authChallengeResponse: isProxy
+                ? { response: 'ProvideCredentials', username: playwrightProxy!.username as string, password: playwrightProxy!.password ?? '' }
+                : { response: 'Default' },
+            });
+          } catch { /* ignore */ }
+          if (isProxy && !proxyAuthed) {
+            proxyAuthed = true;
+            proxyCdp.send('Fetch.disable').catch(() => { });
+          }
+        });
+        logger.info({ server: playwrightProxy.server }, '[crawler] proxy auth via CDP Fetch handler enabled');
+      } catch (e: any) {
+        logger.warn({ err: e?.message }, '[crawler] CDP proxy-auth setup failed — 407 may not resolve');
+      }
+    }
+  } catch {
+    // Non-fatal — crawl proceeds without API endpoint capture
+  }
+  // Final fallback: if rawPage is still null, use the Stagehand page (events may not work)
+  if (!rawPage) rawPage = page;
+
+  try {
+    // Step 1: Navigate and authenticate
+    await onProgress?.(8, 'Loading app');
+    // Use explicit timeout — Stagehand V3Page proxy silently ignores setDefaultNavigationTimeout.
+    // Fall back to 'commit' (HTTP response received) if DOM takes too long; page is still usable.
+    await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(async () => {
+      await page.goto(opts.appUrl, { waitUntil: 'commit', timeout: 15_000 }).catch(() => { });
+    });
+    // Auto-accept cookie consent banners — common EU sites show consent walls before content.
+    await acceptCookieConsent(page, { waitMs: 1_000 }).catch(() => { });
+
+    // Fail fast if the app URL itself is unreachable — datacenter IP block, DNS failure, etc.
+    if (page.url().startsWith('chrome-error://')) {
+      throw new Error(`Cannot reach ${opts.appUrl} — the server refused the connection (likely IP block or DNS failure). Go to Project Settings → Connectivity → Proxy and select "Auto" or enter a residential proxy URL.`);
+    }
+    // Fail fast if Cloudflare or similar bot-protection challenge is served instead of the real app.
+    // These pages never contain testable UI — crawling them produces zero useful screens.
+    const initialTitle = await page.title().catch(() => '');
+    const isBotChallenge = /^(just a moment|attention required|access denied|ddos protection|cloudflare|checking your browser|enable javascript)/i.test(initialTitle);
+    if (isBotChallenge) {
+      throw new Error(`Cannot crawl ${opts.appUrl} — bot protection challenge detected (title: "${initialTitle}"). The site is blocking automated browsers. Go to Project Settings → Connectivity → Proxy and select "Auto" (free proxies) or enter a residential proxy URL to bypass.`);
+    }
+    // Wait for SPA client-side redirect to complete (e.g. Next.js useEffect router.replace to /login).
+    // Without this, loginWithStagehand may run on an empty/loading root page before the form renders.
+    await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
+    await waitForStableDOM(page);
+
+    // Dedicated logged-out/auth discovery phase. This runs before credentials or
+    // storageState are applied, so public pages and auth recovery flows remain
+    // visible even when the main crawl later becomes authenticated.
+    if (!opts.discoveryMode) {
+      try {
+        await runAuthDiscoveryPhase(page, opts, new URL(opts.appUrl).origin, onProgress);
+      } catch (preAuthErr) {
+        logger.warn({ err: preAuthErr }, '[crawler] auth-discovery phase failed — continuing with authenticated crawl');
+      }
+    }
+
+    let didLogin = false; // track whether AI login was actually performed
+
+    if (opts.authToken) {
+      // Auth token already injected via setExtraHTTPHeaders + addInitScript above.
+      // Skip session restore and AI login entirely.
+      await onProgress?.(18, 'API token configured — skipping login');
+      logger.info('authToken provided — skipping AI login');
+    } else if (effectiveStorageState) {
+      // ── StorageState session restore: skip AI login entirely ─────────────────
+      // User authenticated once via `npx playwright codegen --save-storage=session.json`
+      // and saved the resulting JSON in crawl config (encrypted at rest).
+      // effectiveStorageState may also come from Chrome profile inheritance (GAP 3).
+      logger.info('StorageState found — restoring session, skipping AI login');
+      try {
+        const state = JSON.parse(effectiveStorageState!) as {
+          cookies?: Array<Record<string, any>>;
+          origins?: Array<{ origin: string; localStorage: Array<{ name: string; value: string }> }>;
+        };
+
+        if (state.cookies?.length) {
+          // page.context() is not available on Stagehand V3 proxy — use the secondary Playwright client
+          const browserCtx = playwrightBrowser?.contexts()[0];
+          if (browserCtx) {
+            await browserCtx.addCookies(state.cookies as any);
+          }
+        }
+
+        if (state.origins?.length) {
+          await rawPage.addInitScript((origins: typeof state.origins) => {
+            for (const entry of (origins ?? [])) {
+              if (window.location.origin === entry.origin) {
+                for (const item of entry.localStorage) {
+                  try { localStorage.setItem(item.name, item.value); } catch { /* quota/sandboxed */ }
+                }
+              }
+            }
+          }, state.origins);
+        }
+
+        // Reload to apply cookies + trigger localStorage restore initScript
+        await onProgress?.(12, 'Restoring session');
+        await rawPage.reload({ waitUntil: 'domcontentloaded' });
+        await waitForStableDOM(page);
+        // SPA root pages redirect client-side (useEffect → router.replace). Wait for it.
+        if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, '')) {
+          await waitForURLChange(page, opts.appUrl).catch(() => { });
+          await waitForStableDOM(page);
+        }
+
+        // Detect if session restore failed — either by URL redirect or login form presence on root URL
+        const postRestoreUrl = page.url();
+        const LOGIN_PATH = /\/(login|signin|sign-in|auth|log-in|sso|oauth|account\/login)(\/|$|\?)/i;
+        const ERROR_PATH = /\/(session-expired|expired|unauthorized|forbidden|401|403|error|logout|logged-out)(\/|$|\?)/i;
+        const redirectedToLogin = LOGIN_PATH.test(new URL(postRestoreUrl).pathname);
+        const redirectedToError = ERROR_PATH.test(new URL(postRestoreUrl).pathname);
+        // Also check for login form presence — catches apps where login page is at root URL (e.g. /)
+        const hasLoginForm = await page.evaluate(() => {
+          const inputs = Array.from(document.querySelectorAll('input'));
+          const hasEmailOrUser = inputs.some((i: any) =>
+            ['email', 'username', 'user', 'login'].some((k) =>
+              (i.type || '').includes(k) || (i.name || '').toLowerCase().includes(k) ||
+              (i.id || '').toLowerCase().includes(k) || (i.placeholder || '').toLowerCase().includes(k)
+            )
+          );
+          const hasPasswordField = inputs.some((i: any) => i.type === 'password');
+          return hasEmailOrUser && hasPasswordField;
+        }).catch(() => false);
+        const sessionFailed = redirectedToLogin || redirectedToError || hasLoginForm;
+        if (sessionFailed) {
+          logger.warn({ url: postRestoreUrl, loginForm: hasLoginForm, errorUrl: redirectedToError }, 'StorageState restore failed — session may be expired');
+          // Clear the expired session from DB so user is forced to re-capture
+          await AppBrain.saveCrawlConfig(opts.tenantId, opts.projectId, { storageState: null }).catch(() => { });
+          // Navigate to the login page. First try appUrl — if it still redirects to an error/login
+          // path, probe common login paths at the app origin so Stagehand finds a real login form.
+          await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded' }).catch(() => { });
+          await waitForStableDOM(page);
+          const urlAfterNav = page.url();
+          const stillAtError = ERROR_PATH.test(new URL(urlAfterNav).pathname) || LOGIN_PATH.test(new URL(urlAfterNav).pathname);
+          if (stillAtError) {
+            const appOrigin = new URL(opts.appUrl).origin;
+            for (const loginPath of ['/login', '/signin', '/sign-in', '/auth/login', '/auth']) {
+              try {
+                await page.goto(`${appOrigin}${loginPath}`, { waitUntil: 'domcontentloaded' });
+                const hasForm = await page.evaluate(() => {
+                  const inputs = Array.from(document.querySelectorAll('input'));
+                  return inputs.some((i: any) => i.type === 'password');
+                }).catch(() => false);
+                if (hasForm) { logger.info({ loginPath }, 'Found login form at path'); break; }
+              } catch { /* continue to next path */ }
+            }
+            await waitForStableDOM(page);
+          }
+          if (opts.credentials) {
+            logger.info('Falling back to AI login with credentials');
+            await onProgress?.(15, 'Session expired — logging in with credentials…');
+            const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+            if (!loggedIn) throw new Error('[SESSION_EXPIRED] Stored session expired and credential login also failed. Check your username / password in Project Settings → Credentials.');
+            didLogin = true;
+          } else {
+            throw new Error('[SESSION_EXPIRED] Stored session has expired. Please re-capture your session in Project Settings → Advanced: Session Capture, or add username / password as a login fallback.');
+          }
+        } else {
+          await onProgress?.(18, 'Session restored');
+          logger.info({ url: postRestoreUrl }, 'Session restored');
+        }
+      } catch (err) {
+        const msg = String((err as Error).message ?? err);
+        // Re-throw errors we threw intentionally above
+        if (msg.includes('[SESSION_EXPIRED]') || msg.includes('expired')) throw err;
+        logger.warn({ err }, 'StorageState restore failed, falling back to AI login');
+        if (opts.credentials) {
+          await onProgress?.(15, 'Session restore failed — logging in with credentials…');
+          const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+          if (!loggedIn) throw new Error('[SESSION_EXPIRED] Session restore failed and credential login also failed. Check your username / password in Project Settings → Credentials.');
+          didLogin = true;
+        } else {
+          throw new Error('[SESSION_EXPIRED] Session restore failed and no credentials were provided. Please re-capture your session in Project Settings → Advanced: Session Capture.');
+        }
+      }
+    } else if (opts.ssoProvider && opts.ssoDomain && opts.ssoClientId && opts.credentials) {
+      // ── GAP 18: SSO ROPC flow — bypass browser login entirely ────────────────
+      // Get a token directly from the IdP and inject it as a Bearer header.
+      await onProgress?.(12, 'SSO: fetching token from identity provider');
+      try {
+        const ssoToken = await getSsoToken(
+          {
+            provider: opts.ssoProvider as any,
+            domain: opts.ssoDomain,
+            clientId: opts.ssoClientId,
+            clientSecret: opts.ssoClientSecret,
+            scope: opts.ssoScope,
+          },
+          opts.credentials.username,
+          opts.credentials.password
+        );
+        await page.setExtraHTTPHeaders({ 'Authorization': `Bearer ${ssoToken}` });
+        await page.addInitScript((t: string) => {
+          localStorage.setItem('access_token', t);
+          localStorage.setItem('auth_token', t);
+          localStorage.setItem('token', t);
+        }, ssoToken);
+        await page.reload({ waitUntil: 'domcontentloaded' });
+        didLogin = true;
+        await onProgress?.(18, 'SSO token injected');
+        logger.info({ provider: opts.ssoProvider }, '[crawler] SSO token obtained and injected');
+      } catch (ssoErr) {
+        logger.warn({ err: ssoErr }, '[crawler] SSO token retrieval failed — falling back to browser login');
+        // Fall through to browser-based login
+        await onProgress?.(12, 'Logging in');
+        didLogin = true;
+        const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+        if (!loggedIn) {
+          logger.warn('Stagehand login failed — trying Skyvern vision fallback');
+          await onProgress?.(14, 'Trying vision fallback');
+          const skyvernResult = await skyvernFallback({
+            taskDescription: `Log in to ${opts.appUrl} using username "${opts.credentials.username}" and navigate to the main dashboard.`,
+            url: opts.appUrl,
+            credentials: opts.credentials,
+          }, opts.skyvernConfig);
+          if (skyvernResult === 'failed') {
+            throw new Error(
+              'Login failed on both Stagehand (DOM) and Skyvern (vision). ' +
+              'Check credentials. For CAPTCHA/SSO sites, set SKYVERN_API_URL + SKYVERN_API_KEY.'
+            );
+          }
+        }
+        await onProgress?.(18, 'Logged in');
+      }
+    } else if (opts.credentials) {
+      await onProgress?.(12, 'Logging in');
+      didLogin = true;
+      const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+      if (!loggedIn) {
+        logger.warn('Stagehand login failed — trying Skyvern vision fallback');
+        await onProgress?.(14, 'Trying vision fallback');
+        const skyvernResult = await skyvernFallback({
+          taskDescription: `Log in to ${opts.appUrl} using username "${opts.credentials.username}" and navigate to the main dashboard.`,
+          url: opts.appUrl,
+          credentials: opts.credentials,
+        }, opts.skyvernConfig);
+        if (skyvernResult === 'failed') {
+          throw new Error(
+            'Login failed on both Stagehand (DOM) and Skyvern (vision). ' +
+            'Check credentials. For CAPTCHA/SSO sites, set SKYVERN_API_URL + SKYVERN_API_KEY.'
+          );
+        }
+      }
+      await onProgress?.(18, 'Logged in');
+    } else {
+      await onProgress?.(18, 'No auth needed');
+    }
+
+    // Step 2: Crawl
+    // If we just logged in (via credentials or session-fallback), capture and persist the new
+    // session state so the next crawl reuses it without logging in again.
+    if (didLogin) {
+      try {
+        const freshState = await playwrightBrowser!.contexts()[0]?.storageState();
+        if (freshState) {
+          await AppBrain.saveCrawlConfig(opts.tenantId, opts.projectId, { storageState: JSON.stringify(freshState) });
+          logger.info('Fresh session state captured and saved after credential login');
+          await onProgress?.(19, 'Session saved — will reuse on next crawl');
+        }
+      } catch (e) {
+        logger.warn({ err: e }, 'Could not save refreshed session state — crawl will continue');
+      }
+      // Wait for SPA router to complete the post-login redirect before seeding the queue.
+      // Without this, Next.js/React apps queue the login URL instead of the dashboard URL.
+      await waitForURLChange(page, opts.appUrl);
+    }
+    const origin = new URL(page.url()).origin;
+    const maxScreens = opts.maxScreens ?? Infinity;
+    const segmentSize = opts.segmentSize ?? 9_999_999; // effectively unlimited — segments only exist for checkpointing reliability
+    // selective = exact-URL-only mode (recrawl selected screens, no link discovery)
+    const selective = !!(opts.selectiveUrls && opts.selectiveUrls.length > 0);
+
+    // Mark all screens for this project as active — post-crawl we'll diff which ones weren't re-discovered.
+    if (!selective) {
+      await AppBrain.prepareForRecrawl(opts.tenantId, opts.projectId, opts.jobId);
+      await AppBrain.clearFlows(opts.tenantId, opts.projectId);
+    }
+    // Canonicalize URL for dedup: strip tracking params + normalize dynamic ID segments.
+    // Prevents /projects/cuid1 and /projects/cuid2 from being queued as separate screens.
+    const canonicalize = (raw: string): string => {
+      try {
+        const u = new URL(raw);
+        // Treat www and non-www as the same host — most sites redirect one to the other
+        // but internal links may be inconsistent, causing duplicate crawl entries otherwise.
+        u.hostname = u.hostname.replace(/^www\./, '');
+        // Strip params that don't change page content
+        [
+          'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'ref', 'source', '_gl', 'fbclid', 'gclid',
+          'ts', 'timestamp', 'cache', 'cacheBust', 'cache_bust', '_', 'nonce', 'session', 'sid',
+          // WooCommerce / e-commerce action params
+          'add-to-cart', 'add_to_wishlist', 'quantity', 'wc-ajax', 'orderby',
+          'min_price', 'max_price', 'removed_item', 'restore', 'remove_item',
+          'undo_item', 'coupon_code', 'apply_coupon', 'added-to-cart',
+          'wc_error', 'wc_notice', 'wc_notice_type', 'update_cart',
+          'proceed', '_wpnonce', 'post_type', 'product_cat',
+          // WooCommerce facet/layered-nav filter params — each combination generates a unique URL
+          // but all render the same template; not stripping these causes infinite URL explosion
+          // (N filter options × M values = exponential queue growth)
+          'filter_color', 'filter_colour', 'filter_size', 'filter_brand', 'filter_category',
+          'filter_rating', 'filter_stock_status', 'filter_pa_color', 'filter_pa_size',
+          'filter_pa_brand', 'filter_pa_material', 'filter_pa_weight', 'filter_pa_style',
+          'filter_pa_gender', 'filter_pa_age', 'filter_pa_type', 'filter_pa_pattern',
+          'query_type_color', 'query_type_size', 'query_type_brand', 'query_type_pa_color',
+          'query_type_pa_size', 'query_type_pa_brand', 'in_stock', 'on_sale',
+          // Shopify facet filters
+          'sort_by', 'constraint', 'filter.p.tag', 'filter.v.price.gte', 'filter.v.price.lte',
+          'filter.p.m.global.color', 'filter.p.m.global.size',
+        ].forEach(p => u.searchParams.delete(p));
+        // Strip any remaining filter_* / query_type_* / filter.* params (WooCommerce/Shopify facets)
+        for (const key of Array.from(u.searchParams.keys())) {
+          if (/^filter[._]|^query_type_/i.test(key)) u.searchParams.delete(key);
+        }
+        u.searchParams.sort();
+        // Normalize dynamic segments globally: UUIDs, CUIDs, long numeric IDs,
+        // and hyphenated numeric record IDs under any parent route.
+        u.pathname = u.pathname
+          .replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=\/|$)/gi, '/{id}')
+          .replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, '/{id}')
+          .replace(/\/\d{4,}(?=\/|$)/g, '/{id}')
+          // Hyphenated numeric IDs: digits-digits (e.g. 0-9638, 12-4567). Not UUIDs (hex only).
+          .replace(/\/\d+-\d+(?=\/|$)/g, '/{id}')
+          .replace(/\/+$/, '') || '/';
+        // Hash-routed SPAs put the actual route in the fragment ("#/dashboard",
+        // "#!/settings") — keep it so distinct routes dedup separately instead
+        // of every hash-route collapsing onto one canonical (hashless) URL.
+        if (!/^#!?\//.test(u.hash)) u.hash = '';
+        return u.toString();
+      } catch { return raw; }
+    };
+
+    const gitInfo = getGitInfo();
+
+    const visited = new Map<string, string>(); // key = canonical URL
+    // Pre-populate from a previous failed job so already-captured URLs are skipped
+    if (opts.skipUrls?.length) {
+      for (const u of opts.skipUrls) visited.set(canonicalize(u), 'skipped');
+    }
+    let checkpointSuccessCount = 0;
+    let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
+    const startUrl = page.url();
+    // Resume from previous segment's pending queue if provided
+    const resumeUrls = opts.resumeUrls ?? [];
+    // Sitemap-discovered URLs are additional seeds (not a replacement for
+    // link-following) — merged in below unless this is a selective
+    // ("crawl only these URLs") run or a resumed segment, matching how
+    // GraphQL auto-introspection — fire-and-forget, stored in crawl metadata
+    const gqlIntrospectionPromise = (async () => {
+      const gqlEndpoints = guessGraphQLEndpoints(opts.appUrl);
+      for (const ep of gqlEndpoints) {
+        const result = await introspectGraphQL(ep, { timeoutMs: 5_000 }).catch(() => null);
+        if (result?.schema) {
+          logger.info({ endpoint: ep, typeCount: result.schema.types.length }, '[crawler] GraphQL schema discovered');
+          return result;
+        }
+      }
+      return null;
+    })();
+
+    // seedUrls itself works. Best-effort: discoverSitemapUrls() never throws.
+    const sitemapUrls = (opts.useSitemap && !selective && resumeUrls.length === 0)
+      ? await discoverSitemapUrls(opts.appUrl, opts.sitemapUrl)
+      : [];
+    // Filter bad URLs out of resumeUrls (generated dynamic routes from prior segments)
+    const filteredResumeUrls = resumeUrls.filter(u => {
+      try { return !isGeneratedDynamicPath(new URL(u).pathname); } catch { return true; }
+    });
+    const seedUrls = selective
+      ? opts.selectiveUrls!
+      : filteredResumeUrls.length > 0
+        ? filteredResumeUrls
+        : [
+          startUrl,
+          // User-configured seed URLs added to queue; link discovery still runs from each.
+          ...(opts.startUrls ?? []).filter((u) => canonicalize(u) !== canonicalize(startUrl)),
+          ...sitemapUrls.filter((u) => canonicalize(u) !== canonicalize(startUrl)),
+        ];
+    const queued = new Set<string>(seedUrls.map(canonicalize));
+    const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
+    const deduplicator = new ContentDeduplicator(0.88);
+    const deadLinkTracker = new DeadLinkTracker();
+    const redirectRegistry = new RedirectRegistry();
+    // Re-seed from previously-discovered screens so deep routes found by AI link
+    // discovery in prior crawls aren't lost when a recrawl starts from scratch.
+    // Doc 41 Option D: orphan-classified screens (no inbound nav edges) get a BEST_FIRST
+    // score boost via parentElements:0 so they're visited early in the next crawl.
+    // Only applies to full recrawls (not selective runs or segment resumptions).
+    if (!selective && resumeUrls.length === 0) {
+      try {
+        const priorMap = projectForProfile ?? await AppBrain.getProjectMap(opts.tenantId, opts.projectId);
+        const orphanReport = await AppBrain.getOrphanScreens(opts.tenantId, opts.projectId).catch(() => null);
+        const orphanIdSet = new Set<string>(
+          (orphanReport?.screens ?? [])
+            .filter((s: any) => s.classification !== 'REACHABLE')
+            .map((s: any) => s.id as string),
+        );
+        const priorScreens = (priorMap?.screens ?? [])
+          .filter((s: any) => s.status !== 'PENDING_REMOVAL')
+          .filter((s: any) => {
+            const u = s.url as string | null;
+            if (!u) return false;
+            try {
+              const { origin, pathname } = new URL(u);
+              if (origin !== new URL(startUrl).origin) return false;
+              // Don't re-seed generated dynamic routes — they produce duplicate templates.
+              if (isGeneratedDynamicPath(pathname)) return false;
+              return true;
+            } catch { return false; }
+          })
+          .filter((s: any) => !queued.has(canonicalize(s.url as string)));
+        let orphanSeedCount = 0;
+        for (const s of priorScreens) {
+          const u = s.url as string;
+          queued.add(canonicalize(u));
+          const isOrphan = orphanIdSet.has(s.id as string);
+          queue.push(u, isOrphan ? { parentElements: 0 } : undefined);
+          if (isOrphan) orphanSeedCount++;
+        }
+        if (priorScreens.length > 0) {
+          logger.info(
+            { count: priorScreens.length, orphanBoosted: orphanSeedCount },
+            '[crawler] re-seeded prior screen URLs from DB',
+          );
+        }
+      } catch (err) {
+        logger.warn({ err: String(err) }, '[crawler] non-fatal: could not load prior screen URLs for re-seeding');
+      }
+    }
+    // G11: Pre-auth sitemap discovery — capture public pages before session injection.
+    // Many apps have marketing/pricing/docs pages only reachable without auth.
+    // We fetch the sitemap, visit each URL, and screenshot pages that DON'T redirect to login.
+    if (opts.useSitemap !== false) {
+      const g11SitemapUrls: string[] = [];
+      const sitemapCandidates = [
+        `${opts.appUrl.replace(/\/$/, '')}/sitemap.xml`,
+        `${opts.appUrl.replace(/\/$/, '')}/sitemap_index.xml`,
+        `${opts.appUrl.replace(/\/$/, '')}/robots.txt`,
+      ];
+      const LOGIN_REDIRECT_PATH = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
+
+      try {
+        for (const candidate of sitemapCandidates) {
+          try {
+            const resp = await fetch(candidate, { signal: AbortSignal.timeout(8_000) }).catch(() => null);
+            if (!resp?.ok) continue;
+            const text = await resp.text();
+            if (candidate.endsWith('robots.txt')) {
+              const sitemapLines = text.split('\n').filter((l: string) => l.toLowerCase().startsWith('sitemap:'));
+              for (const line of sitemapLines) {
+                const u = line.split(':').slice(1).join(':').trim();
+                if (u) sitemapCandidates.push(u);
+              }
+              continue;
+            }
+            const locs = text.match(/<loc>([^<]+)<\/loc>/gi) ?? [];
+            for (const loc of locs) {
+              const u = loc.replace(/<\/?loc>/gi, '').trim();
+              if (u.startsWith('http')) g11SitemapUrls.push(u);
+            }
+            if (g11SitemapUrls.length > 0) break;
+          } catch { /* try next candidate */ }
+        }
+
+        if (g11SitemapUrls.length > 0) {
+          logger.info({ count: g11SitemapUrls.length }, '[crawler] G11: sitemap discovered public URL candidates');
+          const sitemapOrigin = new URL(opts.appUrl).origin;
+          let sitemapCaptured = 0;
+          for (const sUrl of g11SitemapUrls.slice(0, 30)) {
+            try {
+              const u = new URL(sUrl);
+              if (u.origin !== sitemapOrigin) continue;
+              await page.goto(sUrl, { waitUntil: 'domcontentloaded', timeout: 12_000 });
+              await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
+              await waitForStableDOM(page);
+              const landedUrl = page.url();
+              if (LOGIN_REDIRECT_PATH.test(new URL(landedUrl).pathname)) {
+                const canon = canonicalize(sUrl);
+                if (!queued.has(canon)) { queued.add(canon); queue.push(sUrl); }
+                logger.info({ url: sUrl }, '[crawler] G11: sitemap URL requires auth — added to BFS queue');
+                continue;
+              }
+              const pagePath = u.pathname.replace(/^\//, '') || 'home';
+              const pageName = pagePath.split('/').map((s: string) => s.charAt(0).toUpperCase() + s.slice(1)).join(' ') || 'Home';
+              const shotPath = path.join(opts.screenshotDir ?? '/tmp', `sitemap-preauth-${sitemapCaptured}.png`);
+              await waitForScreenshotReady(page);
+              await page.screenshot({ path: shotPath, fullPage: true }).catch(() => { });
+              const pubScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, pageName, sUrl, opts.jobId, 'SITEMAP').catch(() => null);
+              if (pubScreen) {
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, 'DESKTOP').catch(() => { });
+                logger.info({ url: sUrl, name: pageName }, '[crawler] G11: captured public sitemap page');
+              }
+              sitemapCaptured++;
+            } catch (sitemapPageErr) {
+              logger.warn({ url: sUrl, err: String(sitemapPageErr) }, '[crawler] G11: sitemap page failed — skipping');
+            }
+          }
+          logger.info({ captured: sitemapCaptured }, '[crawler] G11: pre-auth sitemap capture complete');
+          await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => { });
+          await waitForStableDOM(page);
+        }
+      } catch (sitemapErr) {
+        logger.warn({ err: String(sitemapErr) }, '[crawler] G11: pre-auth sitemap discovery failed — continuing');
+      }
+    }
+
+    // Discovery-method tracking for orphan-page detection (see AppBrain.getOrphanScreens):
+    // the entry point is never an "orphan" candidate; anything else in the initial seed
+    // batch is a configured seed URL (sitemap-discovered URLs included), not something
+    // reached by following a link.
+    const canonicalEntryUrl = canonicalize(startUrl);
+    const canonicalSitemapUrls = new Set(sitemapUrls.map(canonicalize));
+    const canonicalSeedUrls = new Set(seedUrls.map(canonicalize));
+    const discoveryMethodFor = (url: string): 'ENTRY_POINT' | 'SEED_URL' | 'SITEMAP' | 'LINK_FOLLOW' => {
+      const c = canonicalize(url);
+      if (c === canonicalEntryUrl) return 'ENTRY_POINT';
+      if (canonicalSitemapUrls.has(c)) return 'SITEMAP';
+      if (canonicalSeedUrls.has(c)) return 'SEED_URL';
+      return 'LINK_FOLLOW';
+    };
+    const edges: Array<{ fromUrl: string; toUrl: string }> = [];
+    const pendingTransitions: Array<{ fromScreenId: string; elementMeaning: string; targetUrl: string }> = [];
+    const discovered: Array<{
+      url: string; screenId?: string; elements?: number;
+      elementFacts?: Array<{ meaning: string; role: string; expectedData?: string }>;
+      screenshotPath?: string; version?: number; error?: string;
+    }> = [];
+    const performanceMetrics: PerformanceMetric[] = [];
+    const seenDomHashes = new Set<string>(opts.knownDomHashes ?? []);
+
+    // Content/legal pages — still discovered as links but skipped from crawl queue
+    // (no test case value; pollute element counts with nav noise)
+    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+    const segIdx = opts.segmentIndex ?? 1;
+
+    const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
+    // AutoThrottle state — exponential moving average of navigation latency
+    let avgNavMs = 0;
+    let navCount = 0;
+    // Rate-limit circuit breaker: after 3 consecutive AI errors (429), skip AI for 10 screens
+    let aiErrorStreak = 0;
+    let aiSkipCount = 0;
+    let aiRateLimitWarned = false;
+    // Hard guard: total URLs attempted (including skips) capped at maxScreens * 4
+    // to prevent infinite-URL-explosion from WooCommerce/Shopify facet filter combos
+    // that slip through canonicalize() if any new filter param type is encountered.
+    let totalAttemptsThisSegment = 0;
+    const maxAttempts = Math.max(segmentSize * 4, 200);
+
+    while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+      if (++totalAttemptsThisSegment > maxAttempts) {
+        logger.warn({ maxAttempts, visited: visited.size, queued: queue.length }, '[crawler] attempt cap reached — stopping to prevent URL explosion');
+        break;
+      }
+      if (opts.abortSignal?.aborted) {
+        logger.info('Abort signal received — stopping crawl');
+        throw new Error('CRAWL_ABORTED');
+      }
+      // AutoThrottle: delay next request proportional to avg navigation latency
+      if (opts.autoThrottle && navCount > 0) {
+        const delayMs = Math.min(5_000, Math.max(0, avgNavMs * 0.5));
+        if (delayMs > 100) await page.waitForTimeout(delayMs).catch(() => { });
+      }
+
+      const url = queue.shift()!;
+      const canonUrl = canonicalize(url);
+      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size + 1}/${segmentSize}` }, '[crawler] visiting URL');
+      const apiObservationStartIndex = capturedApiEndpoints.length;
+
+      // Skip if already visited (canonical match — catches ID-parameterized duplicates)
+      if (visited.has(canonUrl)) continue;
+
+      // robots.txt compliance — skip disallowed URLs
+      if (robotsCache && !await robotsCache.isAllowed(url)) {
+        logger.info({ url }, '[crawler] robots.txt: skipping disallowed URL');
+        continue;
+      }
+
+      // Skip content/legal pages — they have no interactive form elements worth testing
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) {
+          console.log(`[crawler] skip content page: ${url}`);
+          continue;
+        }
+        // Generated ID-like path segments are usually fallback IDs or entity
+        // records. They flood the catalog with duplicate page templates, so
+        // skip them before screenshots or screen creation.
+        if (isGeneratedDynamicPath(pathname)) {
+          console.log(`[crawler] skip generated dynamic route: ${url}`);
+          continue;
+        }
+      } catch { /* invalid url, let it fall through */ }
+
+      try {
+        // Security headers — capture from first document response via CDP.
+        // Stagehand V3's page proxy only supports the 'console' event (page.on throws for others).
+        // We hook directly into the underlying CDP session's Network.responseReceived event instead.
+        let securityHeaders: Record<string, string | null> | null = null;
+        let httpStatus: number | null = null;
+        const urlBase = url.replace(/[?#].*/, '');
+        const cdpSession: any = (page as any).mainSession ?? (page as any)._session ?? null;
+        const secHeadersCdp = (evt: any) => {
+          if (securityHeaders) return;
+          try {
+            const responseUrl: string = evt?.response?.url ?? '';
+            if (!responseUrl.startsWith(urlBase)) return;
+            httpStatus = evt?.response?.status ?? null;
+            const headers: Record<string, string> = {};
+            // CDP headers come as array of {name, value} objects
+            for (const h of (evt?.response?.headers ?? [])) headers[h.name.toLowerCase()] = h.value;
+            securityHeaders = {
+              csp: headers['content-security-policy'] ?? headers['content-security-policy-report-only'] ?? null,
+              hsts: headers['strict-transport-security'] ?? null,
+              xContentType: headers['x-content-type-options'] ?? null,
+              xFrame: headers['x-frame-options'] ?? null,
+              referrer: headers['referrer-policy'] ?? null,
+            };
+          } catch { /* non-fatal */ }
+        };
+        if (cdpSession?.on) cdpSession.on('Network.responseReceived', secHeadersCdp);
+
+        const wsDetector = attachWebSocketDetector(cdpSession);
+
+        if (url !== page.url()) {
+          // LCP/CLS observers are registered via the addInitScript above, which
+          // re-runs automatically on this navigation — nothing to do here.
+          const navStart = Date.now();
+          // Track redirect hops during this navigation
+          const redirectHops: Array<{ from: string; to: string; statusCode: number }> = [];
+          const onRedirectResponse = (res: any) => {
+            const st = res.status();
+            if (st >= 300 && st < 400) {
+              redirectHops.push({ from: res.url(), to: res.headers()['location'] ?? '', statusCode: st });
+            } else if (st >= 400) {
+              deadLinkTracker.record(res.url(), st, { sourceUrl: url });
+            }
+          };
+          rawPage.on('response', onRedirectResponse);
+          await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
+            logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
+          });
+          rawPage.off('response', onRedirectResponse);
+          if (redirectHops.length > 0) {
+            redirectRegistry.register({
+              originalUrl: url, finalUrl: page.url(),
+              hops: redirectHops.map((h, i) => ({ ...h, durationMs: i === 0 ? Date.now() - navStart : 0 })),
+              isCircular: new Set(redirectHops.map(h => h.from)).size < redirectHops.length,
+              hopCount: redirectHops.length,
+              totalDurationMs: Date.now() - navStart,
+            });
+          }
+          // Update exponential moving average for AutoThrottle
+          const navMs = Date.now() - navStart;
+          navCount++;
+          avgNavMs = avgNavMs + (navMs - avgNavMs) / navCount;
+        }
+        // chrome-error:// = TCP/HTTP connection failed (IP block, DNS failure, etc.)
+        // Skip immediately — don't screenshot, don't trigger Skyvern, just move on.
+        if (page.url().startsWith('chrome-error://')) {
+          console.warn(`[crawler] connection failed for ${url} — skipping (chrome-error page)`);
+          continue;
+        }
+        // Full page-ready sequence:
+        // 1. networkidle — waits for XHR/fetch after initial load (data grids, dashboards)
+        // 2. DOM stability — waits for React/Vue hydration to finish mutating the DOM
+        // 3. Modal capture — screenshot app dialogs BEFORE dismissal (they are testable UI states)
+        // 4. Overlay dismissal — removes cookie banners/modals before element extraction
+        // 5. Lazy-load scroll — triggers IntersectionObserver for off-screen images/components
+        await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
+        await waitForStableDOM(page);
+
+        if (!opts.discoveryMode) {
+          // Capture app-generated modals BEFORE dismissing — cookie/GDPR banners excluded.
+          try {
+            const appModalVisible = await page.evaluate(() => {
+              const dialogs = Array.from(document.querySelectorAll('[role="dialog"], [role="alertdialog"], [aria-modal="true"]'));
+              return dialogs.some((el: any) => {
+                const text = (el.textContent ?? '').toLowerCase();
+                const isCookieBanner = /(cookie|consent|gdpr|privacy policy|accept all|we use cookies)/i.test(text);
+                const isVisible = el.offsetParent !== null || el.style.display !== 'none';
+                return isVisible && !isCookieBanner;
+              });
+            }).catch(() => false);
+            if (appModalVisible) {
+              const modalHash = url.replace(/[^a-z0-9]/gi, '').slice(-12);
+              const modalShotPath = path.join(opts.screenshotDir ?? '/tmp', `modal-${modalHash}.png`);
+              await page.screenshot({ path: modalShotPath, fullPage: false }).catch(() => { });
+              const modalName = await page.evaluate(() => {
+                const h = document.querySelector('[role="dialog"] h1,[role="dialog"] h2,[role="dialog"] [role="heading"]');
+                return h?.textContent?.trim() ?? 'Modal';
+              }).catch(() => 'Modal');
+              const modalScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, modalName, url, opts.jobId, discoveryMethodFor(url)).catch(() => null);
+              if (modalScreen) {
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, 'DESKTOP').catch(() => { });
+                logger.info({ url, name: modalName }, '[crawler] captured app modal before dismiss');
+              }
+            }
+          } catch { /* non-fatal */ }
+          await dismissOverlays(page);
+          // Enhanced infinite scroll — handles lazy-load, infinite lists, and virtual scrollers
+          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }).catch(() => { });
+        }
+        if (cdpSession?.off) cdpSession.off('Network.responseReceived', secHeadersCdp);
+        wsDetector.detach();
+
+        // 429 Rate-limit — exponential backoff with jitter before continuing
+        if (httpStatus === 429) {
+          const backoffMs = Math.min(60_000, 1_000 * Math.pow(2, Math.min(navCount, 6)));
+          logger.warn({ url, httpStatus, backoffMs }, '[crawler] 429 rate-limited — backing off');
+          await new Promise(r => setTimeout(r, backoffMs + Math.random() * 1_000));
+        }
+
+        // Skip pages that returned an HTTP 4xx/5xx response — SPAs normally return 200 for all routes,
+        // so a real HTTP error means the server knows this URL is invalid.
+        if (httpStatus !== null && httpStatus >= 400) {
+          logger.info({ url, httpStatus }, '[crawler] skip — server returned HTTP error');
+          continue;
+        }
+
+        // ── Mid-crawl session expiry detection ───────────────────────────────────
+        // Only trigger re-auth when the app REDIRECTED us away from the intended URL.
+        // If we intentionally navigate to /session-expired or /login — save it as a screen (it's
+        // a real page worth testing: messaging, recovery buttons, error UX, etc.).
+        const BFS_ERROR_PATH = /\/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)(\/|$|\?)/i;
+        const BFS_LOGIN_PATH = /\/(login|signin|sign-in|auth|log-in|sso|account\/login)(\/|$|\?)/i;
+        const landedUrl = page.url();
+        const weWantedThisPage = BFS_ERROR_PATH.test(new URL(url).pathname) || BFS_LOGIN_PATH.test(new URL(url).pathname);
+        const redirectedToError = !weWantedThisPage && (
+          BFS_ERROR_PATH.test(new URL(landedUrl).pathname) || BFS_LOGIN_PATH.test(new URL(landedUrl).pathname)
+        );
+        if (redirectedToError) {
+          // We asked for /dashboard but landed on /session-expired — session expired mid-crawl.
+          logger.warn({ requested: url, landed: landedUrl }, '[crawler] Mid-crawl redirect to auth/error — attempting re-auth');
+          let reauthed = false;
+          if (opts.credentials) {
+            reauthed = await loginWithStagehand(
+              stagehand, opts.credentials, opts.appUrl,
+              { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider },
+              { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId },
+              undefined,
+              { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId },
+            ).catch(() => false);
+            if (reauthed) {
+              logger.info({ url }, '[crawler] Re-auth succeeded — retrying URL');
+              await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(() => { });
+              await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
+              await waitForStableDOM(page);
+              const afterReauthUrl = page.url();
+              if (BFS_ERROR_PATH.test(new URL(afterReauthUrl).pathname) || BFS_LOGIN_PATH.test(new URL(afterReauthUrl).pathname)) {
+                logger.warn({ url }, '[crawler] Still redirecting after re-auth — skipping this URL');
+                continue;
+              }
+            } else {
+              logger.warn({ url }, '[crawler] Re-auth failed — stopping BFS (session unrecoverable)');
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                type: 'AUTH_FAILURE',
+                payload: { url, landed: landedUrl, reason: 'Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.' },
+              }).catch(() => { });
+              break;
+            }
+          } else {
+            logger.warn('[crawler] Session expired mid-crawl, no credentials to recover — stopping BFS');
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId,
+              type: 'AUTH_FAILURE',
+              payload: { url, landed: landedUrl, reason: 'Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.' },
+            }).catch(() => { });
+            break;
+          }
+        }
+        // If weWantedThisPage (intentional /login, /session-expired, /404 navigation) —
+        // fall through and save it as a real screen. Error pages are legitimate test targets.
+        if (!redirectedToError && landedUrl !== url) {
+          logger.info({ requested: url, landed: landedUrl }, '[crawler] navigated with redirect');
+        }
+
+        // Discovery mode fast-path: record URL + extract DOM links only; skip screenshots/DB/AI.
+        // IMPORTANT: DOM hash dedup is intentionally DISABLED in discovery mode — SPAs use identical
+        // page shells for every route, so hashing would skip all pages after the first one.
+        if (opts.discoveryMode) {
+          visited.set(canonicalize(url), url); // key=canonical for dedup; value=original URL returned to caller
+
+          // ── SPA hydration wait ───────────────────────────────────────────────────
+          // 'domcontentloaded' fires before JS executes, so shadow DOM doesn't exist yet.
+          // Wait for network idle to ensure JS bundles are downloaded + executed,
+          // then add a brief buffer for component rendering (setTimeout/rAF-deferred work).
+          await (rawPage ?? page).waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
+          await page.waitForTimeout(500).catch(() => { });
+
+          // ── Link extraction — three approaches, merged ───────────────────────────
+          // 1. Playwright native locator (auto-pierces shadow DOM in Playwright 1.20+)
+          //    Works universally for React, Vue, Angular, Polymer, Lit, Stencil, FAST.
+          // 2. BFS shadow DOM traversal via evaluate() — catches any gaps from approach 1.
+          // 3. SPA route manifests (Next.js __NEXT_DATA__, React Router registries).
+          const rawLinksSet = new Set<string>();
+          const addLink = (h: string) => {
+            if (!h || h === '#' || h.startsWith('javascript:') || h.startsWith('mailto:') || h.startsWith('tel:')) return;
+            try {
+              const href = new URL(h, origin).toString();
+              if (isUsefulNavigationUrl(href, origin)) rawLinksSet.add(href);
+            } catch {
+              // ignore malformed links
+            }
+          };
+
+          // 1️⃣ Playwright locator — shadow-DOM-piercing (primary approach)
+          const pwPage = rawPage ?? page;
+          await pwPage.locator('a[href]').evaluateAll((anchors: Element[]) =>
+            anchors.map((a: any) => a.href || a.getAttribute('href') || '')
+          ).then((hrefs: string[]) => hrefs.forEach(addLink)).catch(() => { });
+
+          await pwPage.locator('[data-href],[data-url],[data-link]').evaluateAll((els: Element[]) =>
+            els.map((el: any) => el.dataset?.href || el.dataset?.url || el.dataset?.link || '')
+          ).then((hrefs: string[]) => hrefs.forEach(addLink)).catch(() => { });
+
+          await pwPage.locator('[href]:not(a):not(link):not(base):not(script):not(style)').evaluateAll((els: Element[]) =>
+            els.map((el: any) => el.getAttribute('href') || '')
+          ).then((hrefs: string[]) => hrefs.forEach(addLink)).catch(() => { });
+
+          const playwrightCount = rawLinksSet.size;
+          console.log(`[discover] playwright-locator found ${playwrightCount} links for ${url}`);
+
+          // 2️⃣ BFS shadow DOM evaluate — fallback / complement for stubborn SPAs
+          //    Runs regardless so it can catch elements the locator misses.
+          const evalPage = rawPage ?? page;
+          await evalPage.evaluate((o: string) => {
+            try {
+              const hrefs: string[] = [];
+              const add = (h: string) => {
+                if (h && h !== '#' && !h.startsWith('javascript:') && !h.startsWith('mailto:') && !h.startsWith('tel:')) {
+                  if (h.startsWith(o) || h.startsWith('/')) hrefs.push(h);
+                }
+              };
+              const queue: any[] = [document];
+              const seenRoots = new WeakSet<any>();
+              seenRoots.add(document);
+              let shadowRootsFound = 0;
+              while (queue.length > 0) {
+                const root: any = queue.shift();
+                if (!root) continue;
+                try {
+                  root.querySelectorAll('a[href]').forEach((a: any) => add(a.href || a.getAttribute('href') || ''));
+                  root.querySelectorAll('[data-href],[data-url],[data-link]').forEach((el: any) => {
+                    add((el.dataset && (el.dataset.href || el.dataset.url || el.dataset.link)) || '');
+                  });
+                  root.querySelectorAll('[href]:not(a):not(link):not(base):not(script):not(style)').forEach((el: any) => {
+                    const h = el.getAttribute('href') || '';
+                    if (h && !h.startsWith('#')) add(h);
+                  });
+                  root.querySelectorAll('*').forEach((el: any) => {
+                    try {
+                      if (el.shadowRoot && !seenRoots.has(el.shadowRoot)) {
+                        seenRoots.add(el.shadowRoot);
+                        queue.push(el.shadowRoot);
+                        shadowRootsFound++;
+                      }
+                    } catch (_) { /* closed shadow root — skip */ }
+                  });
+                } catch (_) { /* element threw — skip */ }
+              }
+              return { hrefs, shadowRootsFound };
+            } catch { return { hrefs: [] as string[], shadowRootsFound: 0 }; }
+          }, origin).then((result: any) => {
+            (result?.hrefs ?? []).forEach(addLink);
+            if (result?.shadowRootsFound > 0) {
+              console.log(`[discover] bfs found ${result.shadowRootsFound} shadow roots, added ${rawLinksSet.size - playwrightCount} extra links for ${url}`);
+            }
+          }).catch(() => { });
+
+          // 3️⃣ SPA route manifests
+          await evalPage.evaluate(() => {
+            const routes: string[] = [];
+            try {
+              const nd = (window as any).__NEXT_DATA__;
+              if (nd?.page) routes.push(nd.page);
+            } catch (_) { /* non-fatal */ }
+            try {
+              const w = window as any;
+              const rr = w.__reactRouterRoutes || (w.__reactRouterManifest?.routes);
+              if (rr) {
+                const collect = (r: any) => {
+                  if (r?.path && r.path !== '*') routes.push(r.path.startsWith('/') ? r.path : '/' + r.path);
+                  if (r?.children) r.children.forEach(collect);
+                };
+                (Array.isArray(rr) ? rr : Object.values(rr)).forEach(collect);
+              }
+            } catch (_) { /* non-fatal */ }
+            return routes;
+          }).then((routes: any[]) => {
+            (routes ?? []).forEach(p => {
+              try { addLink(new URL(p, origin).href); } catch { /* ignore */ }
+            });
+          }).catch(() => { });
+
+          const rawLinks = Array.from(rawLinksSet);
+          console.log(`[discover] ${url} → rawLinks=${rawLinks.length} origin=${origin} pageUrl=${page.url()}`);
+          if (rawLinks.length > 0) console.log(`[discover] sample links: ${rawLinks.slice(0, 5).join(' | ')}`);
+          let newlyQueued = 0;
+          for (const link of rawLinks) {
+            try {
+              const resolved = link.startsWith('/') ? `${origin}${link}` : link;
+              const u = new URL(resolved);
+              // Keep hash-router fragments (#/route, #!/route) — strip plain anchors (#section)
+              if (!/^#!?\//.test(u.hash)) u.hash = '';
+              const clean = u.toString();
+              if (!isUsefulNavigationUrl(clean, origin)) continue;
+              // Dedup: use canonical form to avoid crawling 1000 product detail pages separately
+              const norm = canonicalize(clean);
+              if (!visited.has(norm) && !queued.has(norm)) { queued.add(norm); queue.push(clean); newlyQueued++; }
+            } catch { /* ignore malformed */ }
+          }
+          if (newlyQueued > 0) console.log(`[discover] queued ${newlyQueued} new URLs, queue size=${queue.length}`);
+          const pct = 10 + Math.round((visited.size / Math.max(1, opts.maxScreens ?? 1000)) * 80);
+          await onProgress?.(Math.min(pct, 89), `Discovered ${visited.size} URLs…`);
+          // Stream partial results to caller so they can be persisted/shown incrementally
+          if (opts.onUrlsDiscovered) {
+            await opts.onUrlsDiscovered(Array.from(visited.values())).catch(() => { });
+          }
+          continue;
+        }
+
+        // Normal crawl mode: DOM hash dedup — detect duplicate-structure pages (SPA 404s, template clones).
+        const preDomHash = await computeDomHash(page);
+        if (seenDomHashes.has(preDomHash)) {
+          console.log(`[crawler] skip ${url} — duplicate DOM structure (same content as prior page)`);
+          continue;
+        }
+        const pageTitle = await page.title().catch(() => '');
+        if (/\b(not found|404|page not found|this page could not be found)\b/i.test(pageTitle)) {
+          console.log(`[crawler] skip ${url} — error page detected (title: "${pageTitle}")`);
+          seenDomHashes.add(preDomHash); // register so future 404 pages also hit hash-dedup
+          continue;
+        }
+        // Do NOT add real pages to seenDomHashes — server-rendered sites (WooCommerce, WordPress)
+        // use the same template for many legitimate pages; registering the hash would skip them all.
+
+        // Collect Navigation Timing + Paint Timing immediately after DOM stable.
+        // LCP/CLS come from the buffered PerformanceObservers registered in the
+        // init script above (the synchronous timeline never holds LCP entries).
+        const rawPerf = await page.evaluate(() => {
+          const nav = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming | undefined;
+          const paints = performance.getEntriesByType('paint');
+          // Prefer pre-registered observer value (window.__zetaLCP) over buffered entries.
+          // Buffered entries are often empty if the observer wasn't registered before navigation.
+          const lcpFromObserver: number | null = (window as any).__zetaLCP ?? null;
+          const lcpEntries = performance.getEntriesByType('largest-contentful-paint');
+          const lcpFromEntries = lcpEntries.length > 0
+            ? Math.round((lcpEntries[lcpEntries.length - 1] as any).startTime)
+            : undefined;
+          // @ts-ignore — stashed by the CLS init script
+          // @ts-ignore — stashed by the web-vitals init script
+          const stashedLcp: number | null = (window as any).__zetaLCP ?? null;
+          // @ts-ignore
+          const stashedCls: number | undefined = (window as any).__zetaCLS;
+          return {
+            ttfb: nav ? Math.round(nav.responseStart - nav.fetchStart) : undefined,
+            fcp: paints.find(p => p.name === 'first-contentful-paint')?.startTime
+              ? Math.round(paints.find(p => p.name === 'first-contentful-paint')!.startTime)
+              : undefined,
+            lcp: lcpFromObserver != null ? Math.round(lcpFromObserver) : lcpFromEntries,
+            cls: typeof stashedCls === 'number' ? Math.round(stashedCls * 1000) / 1000 : undefined,
+          };
+        }).catch(() => null);
+
+        // Accessibility audit via axe-core (WCAG 2.0 A/AA + best-practice)
+        const a11yResult = await (async () => {
+          try {
+            const _require = createRequire(import.meta.url);
+            const axePath = _require.resolve('axe-core');
+            const axeScript = readFileSync(axePath, 'utf8');
+            await page.addScriptTag({ content: axeScript });
+            const result = await page.evaluate(async () => {
+              return (window as any).axe.run(document, {
+                runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'best-practice'] },
+                resultTypes: ['violations'],
+              });
+            });
+            // Full violation objects for DB storage (Phase 4.4)
+            const violations = result.violations.map((v: any) => ({
+              impact: v.impact as string,
+              rule: v.id as string,
+              description: v.description as string,
+              // First offending node's outer HTML (truncated later in saveA11yViolations)
+              nodeHtml: (v.nodes?.[0]?.html as string | undefined) ?? undefined,
+              wcagTags: (v.tags as string[]).filter((t: string) => t.startsWith('wcag') || t.startsWith('best-')),
+            }));
+            const criticalCount = violations.filter((v: any) => v.impact === 'critical').length;
+            return { violations, violationCount: violations.length, criticalCount };
+          } catch { return null; }
+        })();
+        const a11yScore = a11yResult
+          ? Math.max(0, 100 - (a11yResult.criticalCount * 20) - ((a11yResult.violationCount - a11yResult.criticalCount) * 5))
+          : null;
+
+
+        // Screen name strategy (SPA-aware):
+        //   1. Title with page-specific segment (e.g. "Dashboard | ZeTA")
+        //   2. aria-current="page" active nav item (reliable for SPAs)
+        //   3. Page-specific headings that are NOT the brand name
+        //   4. URL path segments (skipping cuid/UUIDs/numerics)
+        const name = await (async () => {
+          let brandName = '';
+
+          // Compute reliable URL path-based name — skip cuid/UUID/numeric segments
+          let pathName = 'Home';
+          try {
+            const u = new URL(url);
+            const hostname = u.hostname.replace(/^www\./, '').split('.')[0]!.toLowerCase();
+            const allSegs = u.pathname.split('/').filter(Boolean);
+            const IS_ID = (s: string) =>
+              /^\d+$/.test(s) ||                   // numeric
+              /^[0-9a-f]{8,}$/i.test(s) ||         // hex id
+              /^[a-z]{1,2}[0-9a-z]{20,}$/i.test(s); // cuid/cuid2 (e.g. cm...)
+
+            const readable = allSegs
+              .filter((s) => !IS_ID(s))
+              .map((s) => s.charAt(0).toUpperCase() + s.slice(1).replace(/[-_]/g, ' '));
+
+            if (readable.length > 0) {
+              const last = readable[readable.length - 1]!;
+              const GENERIC = ['edit', 'new', 'create', 'view', 'show', 'index', 'detail', 'details'];
+              if (GENERIC.includes(last.toLowerCase()) && readable.length > 1) {
+                pathName = `${readable[readable.length - 2]} · ${last}`;
+              } else {
+                pathName = last;
+              }
+            } else if (allSegs.length > 0) {
+              // All segments were IDs — use the last non-trivial one
+              const lastSeg = allSegs[allSegs.length - 1]!;
+              pathName = lastSeg.slice(0, 8).toUpperCase();
+            }
+
+            // Append significant query params
+            const SIG_PARAMS = ['product_cat', 'category', 'cat', 'type', 'tag', 'brand', 'filter', 'tab', 'section', 'view'];
+            const paramSuffix = SIG_PARAMS
+              .map((p) => { const v = u.searchParams.get(p); return v ? ` · ${v.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())}` : ''; })
+              .find(Boolean) ?? '';
+            pathName = pathName + paramSuffix;
+
+            // Detect brand name from hostname for later comparisons
+            brandName = hostname;
+          } catch { /* fall through */ }
+
+          // 1. Document title — use only if it has a page-specific segment
+          try {
+            const title = await page.title();
+            if (title && title.length > 0 && title.length < 120) {
+              const parts = title.split(/\s*[|—\-–·]\s*/);
+              // Store brand candidate (usually last segment)
+              if (parts.length > 1) brandName = (parts[parts.length - 1] ?? brandName).trim().toLowerCase();
+
+              if (parts.length > 1) {
+                // "Dashboard | ZeTA" → "Dashboard"
+                const candidate = parts[0].trim();
+                if (candidate.length > 2 && !/^\d[\d\s]*$/.test(candidate)) return candidate;
+              }
+              // Single-segment title — only use if multi-word and not brand-name-like
+              const candidate = parts[0].trim();
+              const isBrand = candidate.split(/\s+/).length === 1 &&
+                (candidate.toLowerCase().includes(brandName) || candidate.toLowerCase() === brandName);
+              if (!isBrand && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
+                return candidate;
+              }
+            }
+          } catch { /* fall through */ }
+
+          // 2. Active nav item — SPAs set aria-current="page" on the active route link
+          try {
+            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: 2000 }).catch(() => null);
+            if (activeNav) {
+              const t = activeNav.trim();
+              if (t.length > 1 && t.length < 60 && t.toLowerCase() !== brandName) return t;
+            }
+          } catch { /* fall through */ }
+
+          // 3. Page-specific heading — skip if it's just the brand name
+          try {
+            const selectors = [
+              'main h1', '[role="main"] h1', '.page-title', '.page-header h1', '.content h1',
+              'main h2', '[role="main"] h2', '.page-header h2',
+            ];
+            for (const sel of selectors) {
+              const h = await page.locator(sel).first().textContent({ timeout: 2000 }).catch(() => null);
+              if (!h) continue;
+              const t = h.trim();
+              if (t.length < 2 || t.length > 80) continue;
+              // Skip if it's the brand name (exact or contains hostname)
+              if (t.toLowerCase() === brandName) continue;
+              if (brandName && t.toLowerCase().replace(/\s/g, '').includes(brandName.replace(/\s/g, ''))) continue;
+              return t;
+            }
+          } catch { /* fall through */ }
+
+          // Safety: if the only candidate is the brand/hostname, fall back to URL path.
+          try {
+            const hostname = new URL(url).hostname.replace(/^www\./, '').split('.')[0]!.toLowerCase();
+            if (pathName !== 'Home' || name.toLowerCase() === hostname || name.toLowerCase().replace(/\s/g, '') === hostname) {
+              // name is still brand-name-equivalent — use URL-derived pathName
+              if (name.toLowerCase() === hostname || name.toLowerCase().replace(/\s/g, '') === hostname) {
+                return pathName;
+              }
+            }
+          } catch { /* ignore */ }
+
+          return pathName;
+        })();
+        const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
+        visited.set(canonicalize(url), screen.id);
+
+        const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
+
+        // Wait for skeleton loaders / aria-busy spinners to resolve before screenshotting and analysis.
+        // Stagehand V3 proxy does NOT support page.waitForFunction — use page.evaluate polling instead.
+        await (async () => {
+          const deadline = Date.now() + 4_000;
+          while (Date.now() < deadline) {
+            const ready = await page.evaluate(() => {
+              const busy = document.querySelectorAll('[aria-busy="true"], [data-loading="true"]');
+              const skeletons = document.querySelectorAll('.skeleton, [class*="skeleton"], [class*="Skeleton"]');
+              return busy.length === 0 && skeletons.length === 0;
+            }).catch(() => true); // if evaluate fails, assume ready
+            if (ready) break;
+            await new Promise(r => setTimeout(r, 200));
+          }
+        })();
+
+        // Fire-and-forget per-page analyses — non-blocking, failures silently swallowed.
+        // screen is now declared; page still has current page state and is fully hydrated.
+        void (async () => {
+          const screenId = screen.id;
+          const [seoR, contrastR, flagsR, langR, formsR] = await Promise.allSettled([
+            checkSeo(page, url),
+            auditColorContrast(page, url),
+            detectFeatureFlags(page, url),
+            detectLanguage(page, url),
+            detectForms(page),
+          ]);
+          const base = { screenId, jobId: opts.jobId };
+          if (seoR.status === 'fulfilled')
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'SEO_CHECK', payload: seoR.value }).catch(() => { });
+          if (contrastR.status === 'fulfilled' && (contrastR.value as any).issues?.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'COLOR_CONTRAST', payload: contrastR.value }).catch(() => { });
+          if (flagsR.status === 'fulfilled' && (flagsR.value as any).flags?.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'FEATURE_FLAGS', payload: flagsR.value }).catch(() => { });
+          if (langR.status === 'fulfilled')
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'LANGUAGE_DETECTION', payload: langR.value }).catch(() => { });
+          if (formsR.status === 'fulfilled' && Array.isArray(formsR.value) && formsR.value.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'FORMS_DETECTED', payload: { forms: formsR.value } }).catch(() => { });
+        })();
+
+        // Freeze all CSS animations/transitions so screenshots are stable across runs
+        try { await page.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; }' }); } catch { /* Stagehand proxy may not expose addStyleTag */ }
+
+        // Replay-first: check for a cached link-discovery recipe before calling AI Stagehand
+        const canonUrl = canonicalize(url);
+        const linkRecipe = await AppBrain.getActionRecipe(
+          opts.tenantId, opts.projectId, new URL(url).hostname, `link-discovery:${canonUrl}`
+        ).catch(() => null);
+        const cachedLinks: string[] | null = (linkRecipe && linkRecipe.successRate >= 0.7)
+          ? ((linkRecipe.steps as any[])
+            .map((s: any) => s.href)
+            .filter((href: string | undefined): href is string => !!href && isUsefulNavigationUrl(href, origin)))
+          : null;
+        if (cachedLinks) {
+          logger.info({ url, cachedCount: cachedLinks.length }, '[crawler] link-discovery: using cached recipe — AI will still run and merge');
+        }
+
+        // Emit progress now — navigation is done and the page is stable.
+        // This fires before the 30-90s parallel screenshot+AI phase, preventing a silent gap in the UI.
+        await onProgress?.(20, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
+
+        // P3a: Async content detection — get raw DOM element count before AI extraction.
+        // The AI extraction takes 30-90s; any async-loaded content will be visible after it completes.
+        // Compare pre vs post count to detect dashboards that populate after initial load.
+        const preDomCount = await page.evaluate(() =>
+          document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
+        ).catch(() => 0);
+
+        await waitForScreenshotReady(page);
+
+        // Parallelize: screenshot + element extraction + link discovery + AT capture (all work on current page state)
+        const [, elements, nextLinks, atElements] = await Promise.all([
+          page.screenshot({ path: shotPath, fullPage: true })
+            .then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP')),
+          extractElements(stagehand, page, elementLLM ?? undefined).catch(() => [] as Awaited<ReturnType<typeof extractElements>>),
+          // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
+          // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0)
+            .then((freshLinks) => {
+              if (aiSkipCount > 0) aiSkipCount--;
+              aiErrorStreak = 0;
+              const merged = (cachedLinks ? Array.from(new Set([...cachedLinks, ...freshLinks])) : freshLinks)
+                .filter((href) => isUsefulNavigationUrl(href, origin));
+              logger.info({ url, domLinks: freshLinks.length, cached: cachedLinks?.length ?? 0, merged: merged.length }, '[crawler] link-discovery: result');
+              // Always persist fresh recipe so stale cache gets overwritten with new AI discoveries
+              if (merged.length > 0) {
+                AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
+                  domain: new URL(url).hostname,
+                  purpose: `link-discovery:${canonUrl}`,
+                  fingerprint: sha256(canonUrl),
+                  steps: merged.map((href) => ({ action: 'link', href })),
+                  successRate: 0.8,
+                }).catch(() => { });
+              }
+              return merged;
+            })
+            .catch((err: any) => {
+              const fallback: string[] = err?.domLinks ?? [];
+              if (err?.isRateLimit) {
+                aiErrorStreak++;
+                if (aiErrorStreak >= 3) { aiSkipCount = 10; logger.warn('[crawler] 3 consecutive AI rate limits — skipping AI link discovery for next 10 screens'); }
+                if (!aiRateLimitWarned) {
+                  aiRateLimitWarned = true;
+                  onProgress?.(20, 'AI quota exhausted — falling back to DOM-only link discovery').catch(() => { });
+                }
+              }
+              return fallback;
+            }),
+          readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>),
+        ]);
+        logger.info({ url, screenId: screen.id, version: screen.version, elements: elements.length, links: nextLinks.length, atSample: atElements.slice(0, 3).map((a: any) => a.label) }, '[crawler] screen captured');
+        console.log('[crawler] AT:', JSON.stringify(atElements.slice(0, 5)));
+
+        // Capture DOM snapshot for structural analysis, and extract Markdown/text
+        // content from the same DOM (crawl4AI-parity text output) — no extra fetch.
+        try {
+          const domContent = await capturePageHtml(page);
+          if (!domContent) {
+            logger.warn({ url, screenId: screen.id }, '[crawler] markdown extraction skipped: DOM snapshot unavailable');
+          } else {
+            const domPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+            await fs.writeFile(domPath, domContent, 'utf8');
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath).catch(() => { });
+
+            // Supplement markdown with iFrame and Shadow DOM content
+            const [iframeContent, shadowContent] = await Promise.all([
+              extractIframeContent(page).catch(() => []),
+              pierceShadowDom(page).catch(() => ''),
+            ]);
+            const supplemental = [
+              ...iframeContent.map((f: any) => f.markdown).filter(Boolean),
+              shadowContent,
+            ].join('\n\n').trim();
+
+            const extracted = extractMarkdown(
+              supplemental ? domContent + '\n<!-- iframe/shadow -->\n' + supplemental.slice(0, 3_000) : domContent,
+              url
+            );
+            let fitMarkdown: string | undefined;
+            if (extracted) {
+              const mdPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.md');
+              fitMarkdown = buildFitMarkdown(extracted.markdown, [
+                ...domainProfile.importantScreenPatterns,
+                ...domainProfile.workflowGoals,
+                ...(opts.bestFirstKeywords ?? []),
+              ]);
+              const { text: scrubbed, scrubbed: hadPii } = scrubPii(extracted.markdown);
+              if (hadPii) logger.info({ url }, '[crawler] PII detected and redacted from markdown');
+              const dedupResult = deduplicator.check(url, scrubbed);
+              if (dedupResult.isDuplicate) {
+                logger.info({ url, similarUrl: dedupResult.similarUrl, score: dedupResult.similarityScore }, '[crawler] near-duplicate page skipped');
+              }
+              await fs.writeFile(mdPath, scrubbed, 'utf8');
+
+              // Generate a one-sentence AI summary from the markdown — fire-and-forget
+              const summaryPromise = (async () => {
+                try {
+                  const llmCfg = await getTaskAgentLLMConfig(opts.tenantId, 'crawl').catch(() => undefined);
+                  const snippet = extracted.markdown.slice(0, 1200);
+                  const { text, usage } = await askTracked(
+                    'You are a QA analyst. Given page content, write ONE crisp sentence describing what this screen does or shows. Be specific. No preamble.',
+                    `Screen: "${name}"\nURL: ${url}\n\nContent:\n${snippet}`,
+                    llmCfg,
+                    80,
+                  );
+                  if (usage) {
+                    recordLLMCall({
+                      tenantId: opts.tenantId,
+                      projectId: opts.projectId,
+                      jobId: opts.jobId,
+                      agentType: 'crawl_summary',
+                      label: `page_summary:${name}`,
+                      usage,
+                    }).catch(() => { });
+                  }
+                  return text.trim().replace(/^["']|["']$/g, '');
+                } catch { return undefined; }
+              })();
+
+              AppBrain.updateScreenMarkdown(opts.tenantId, opts.projectId, screen.id, {
+                markdownPath: mdPath,
+                markdownMethod: extracted.method,
+                markdownWordCount: extracted.wordCount,
+                markdown: fitMarkdown ?? extracted.markdown,
+                heading: extracted.title || name,
+              }).catch((err: any) => console.warn('[crawler] markdown extraction save failed:', err));
+
+              summaryPromise.then((aiSummary) => {
+                if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => { });
+              });
+            }
+
+            const structuredJson = summarizeStructuredElements(atElements);
+            const artifact = buildCrawlArtifactEnvelope({
+              tenantId: opts.tenantId,
+              projectId: opts.projectId,
+              jobId: opts.jobId,
+              url,
+              canonicalUrl: canonicalize(url),
+              screenId: screen.id,
+              screenshotPath: shotPath,
+              domPath,
+              domContent,
+              markdown: extracted?.markdown,
+              fitMarkdown,
+              accessibilityElements: atElements,
+              structuredJson,
+              evidence: [
+                { type: 'domain_profile', payload: { industry: domainProfile.industry, confidence: domainProfile.confidence } },
+                { type: 'engine_route', payload: routeDecision },
+                { type: 'screen_fingerprints', payload: { urlCanonicalHash: sha256(canonicalize(url)) } },
+              ],
+            });
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              screenId: screen.id,
+              jobId: opts.jobId,
+              type: 'SCREEN_ARTIFACT_ENVELOPE',
+              payload: artifact,
+            }).catch((err: any) => console.warn('[crawler] artifact evidence save failed:', err?.message ?? err));
+          }
+        } catch (err: any) {
+          logger.warn({ url, screenId: screen.id, err: String(err?.message ?? err) }, '[crawler] markdown extraction failed');
+        }
+
+        // Persist a11y score + individual violations — fire-and-forget, non-fatal
+        if (a11yScore !== null) {
+          AppBrain.updateScreenA11y(opts.tenantId, screen.id, { a11yScore })
+            .catch((err: any) => console.warn('[crawler] a11y score update failed:', err));
+        }
+        if (a11yResult !== null && a11yResult.violations.length > 0) {
+          AppBrain.saveA11yViolations(opts.tenantId, opts.projectId, screen.id, a11yResult.violations, opts.jobId)
+            .catch((err: any) => console.warn('[crawler] a11y violations save failed:', err));
+        }
+
+        // DOM hash was registered before saveScreen — no action needed here.
+
+        // Build a role→selectorHint lookup from AT data so fingerprints can include tag hints
+        const atByRole = new Map<string, string>();
+        for (const at of atElements) {
+          if (!atByRole.has(at.role)) atByRole.set(at.role, at.selectorHint);
+        }
+
+        // Auto-create DESKTOP visual baseline on first crawl (no-op on recrawls)
+        AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, screen.id, 'DESKTOP', shotPath, gitInfo.branch, gitInfo.commit || undefined)
+          .catch(() => { });
+
+        // Phase 2: compute element hash for stale-test detection
+        const newElementHash = computeElementHash(elements);
+        AppBrain.updateElementHash(opts.tenantId, screen.id, newElementHash, (screen as any).elementHash ?? null)
+          .catch((err: any) => console.warn('[crawler] element hash update failed:', err));
+
+        // Phase 3: perceptual screenshot diff — non-blocking
+        if (screen.version > 1) {
+          (async () => {
+            try {
+              const prevShot = await AppBrain.getPreviousScreenshotPath(opts.tenantId, screen.id, screen.version - 1);
+              if (prevShot) {
+                const { screenshotDiffPercent } = await import('./phash.js');
+                const { pHashDistance, changeLevel } = await screenshotDiffPercent(prevShot, shotPath);
+                await AppBrain.saveScreenDiff(opts.tenantId, opts.projectId, screen.id, screen.version - 1, screen.version, pHashDistance, changeLevel);
+              }
+            } catch (err) {
+              console.warn('[crawler] pHash diff skipped:', err);
+            }
+          })();
+        }
+
+        // VisualDiff pipeline — compare against approved baseline (not previous screenshot)
+        if (screen.version > 1) {
+          (async () => {
+            try {
+              const baseline = await AppBrain.getVisualBaselineForDiff(opts.tenantId, screen.id, 'DESKTOP', gitInfo.branch);
+              if (baseline && baseline.storagePath !== shotPath) {
+                const { screenshotDiffPercent, gridLayoutDiff } = await import('./phash.js');
+                const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
+                  screenshotDiffPercent(baseline.storagePath, shotPath, baseline.ignoredRegions),
+                  gridLayoutDiff(baseline.storagePath, shotPath, baseline.ignoredRegions),
+                  AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, 'DESKTOP'),
+                ]);
+                let verdict = pHashVerdict(pHashDistance, changeLevel, [
+                  ...layoutChanges,
+                  ...elementShifts.map((s) => ({ location: 'element', severity: s.severity, description: s.description })),
+                ]);
+
+                // pHash is a cheap, always-on filter; only spend an LLM call on the
+                // screens it flags as worth a closer look. It also can't tell
+                // meaningful UI changes apart from anti-aliasing/font-rendering noise,
+                // which is exactly what Visual AI comparison exists to filter out.
+                if (verdict.changeType === 'SIGNIFICANT' || verdict.changeType === 'CRITICAL') {
+                  try {
+                    const regions = baseline.ignoredRegions ?? [];
+                    // Same mask the pHash passes already applied — otherwise a
+                    // manually-masked region (timestamp, ad banner, live counter)
+                    // still reaches the AI verdict unmasked and can get flagged.
+                    const [baselineBuffer, newBuffer] = regions.length > 0
+                      ? await (async () => {
+                        const { applyMask } = await import('./phash.js');
+                        return Promise.all([applyMask(baseline.storagePath, regions), applyMask(shotPath, regions)]);
+                      })()
+                      : await Promise.all([
+                        fs.readFile(baseline.storagePath),
+                        fs.readFile(shotPath),
+                      ]);
+                    const aiResult = await aiVisualCompare({
+                      baselineImageBase64: baselineBuffer.toString('base64'),
+                      newImageBase64: newBuffer.toString('base64'),
+                      screenName: screen.name,
+                      screenPurpose: 'Application screen captured during automated crawl',
+                      tenantId: opts.tenantId,
+                      projectId: opts.projectId,
+                    });
+                    verdict = mergeAiVisualVerdict(verdict, aiResult, screen.name);
+                  } catch (aiErr) {
+                    logger.warn({ screenId: screen.id, err: String(aiErr) }, '[crawler] AI visual compare failed, keeping pHash-only verdict');
+                  }
+                }
+
+                const { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+                if (changeType !== 'NONE') {
+                  let newScreenshotCloudUrl: string | undefined;
+                  try {
+                    const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import('@detiq/app-brain');
+                    const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
+                    if (storageCfg) {
+                      const key = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
+                      newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);
+                    }
+                  } catch { /* non-fatal — falls back to local path */ }
+                  await AppBrain.saveVisualDiff(opts.tenantId, {
+                    baselineId: baseline.id,
+                    newScreenshotPath: shotPath,
+                    newScreenshotCloudUrl,
+                    diffScore: pHashDistance,
+                    changeType,
+                    changes: allChanges,
+                    recommendation,
+                    bugReportDraft,
+                    aiSummary,
+                  });
+                }
+              }
+            } catch (err: any) {
+              const missing = String(err?.message ?? err).includes('Input file is missing');
+              if (missing) {
+                logger.warn({ screenId: screen.id, version: screen.version, err: String(err?.message ?? err) }, '[crawler] VisualDiff skipped — baseline screenshot not on disk (stored in cloud only). Run a full recrawl to regenerate baselines.');
+              } else {
+                console.warn('[crawler] VisualDiff pipeline failed:', err);
+              }
+            }
+          })();
+        }
+
+        const linkMappings = await extractLinkMappings(page, origin);
+        for (const el of elements) {
+          const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark);
+          // Populate ElementFingerprint — fire-and-forget, failures are non-fatal
+          const label = el.meaning;
+          const role = el.role ?? 'unknown';
+          const selectorHint = atByRole.get(role) ?? role;
+          const selectors = [
+            { type: 'role-label', value: `${role}:${label}`, confidence: 0.9 },
+            { type: 'aria-label', value: label, confidence: 0.85 },
+            { type: 'tag-hint', value: selectorHint, confidence: 0.6 },
+          ];
+          AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+        }
+        for (const lm of linkMappings) {
+          const safety = assessActionSafety(lm.label, domainProfile);
+          if (!safety.allowed) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              screenId: screen.id,
+              jobId: opts.jobId,
+              type: 'ACTION_BLOCKED_BY_SAFETY_POLICY',
+              payload: { label: lm.label, href: lm.href, decision: safety, profile: domainProfile.industry },
+            }).catch(() => { });
+            continue;
+          }
+          pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
+          AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
+            domain: new URL(url).hostname,
+            purpose: `navigate:${lm.label}`,
+            fingerprint: sha256(`${canonicalize(url)}:${lm.label}:${canonicalize(lm.href)}`),
+            steps: [{ action: 'click', label: lm.label, targetUrl: lm.href, fromUrl: url }],
+            successRate: 0.5,
+          }).catch(() => { });
+
+          if (!selective) {
+            try {
+              const targetPath = new URL(lm.href).pathname;
+              const canonLmHref = canonicalize(lm.href);
+              if (
+                !SKIP_PATH.test(targetPath) &&
+                isUsefulNavigationUrl(lm.href, origin) &&
+                !queued.has(canonLmHref) &&
+                !visited.has(canonLmHref) &&
+                queued.size < maxScreens * 2
+              ) {
+                if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
+                queued.add(canonLmHref);
+                queue.push(lm.href, { parentElements: elements.length });
+                edges.push({ fromUrl: url, toUrl: lm.href });
+                logger.info({ fromUrl: url, newUrl: lm.href, label: lm.label, queueSize: queue.length }, '[crawler] anchor mapping added to queue');
+              }
+            } catch { /* ignore invalid link mapping */ }
+          }
+        }
+        if (!selective) {
+          for (const href of nextLinks) {
+            edges.push({ fromUrl: url, toUrl: href });
+            // Skip WooCommerce/e-commerce action URLs — these are AJAX/form actions that render
+            // the same page template with a compare widget; they are not distinct page routes.
+            // They would all get queued (different `id` param = different canonical URL), navigated,
+            // found "duplicate DOM", and burn through totalAttemptsThisSegment without any gain.
+            try {
+              const hrefParams = new URL(href).searchParams;
+              const actionVal = hrefParams.get('action') ?? '';
+              if (/^yith-|^wc-|^vc_|^elementor/.test(actionVal)) continue;
+            } catch { /* invalid URL, let it fall through */ }
+            const canonHref = canonicalize(href);
+            if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 2) {
+              // Don't flood the queue with generated ID-like route segments.
+              try {
+                const hrefPath = new URL(href).pathname;
+                if (isGeneratedDynamicPath(hrefPath)) continue;
+              } catch { /* ignore invalid URLs */ }
+              if (robotsCache && !await robotsCache.isAllowed(href)) continue;
+              queued.add(canonHref);
+              queue.push(href, { parentElements: elements.length }); // navigate to actual URL, dedup by canonical
+              logger.info({ newUrl: href, queueSize: queue.length }, '[crawler] new URL added to queue');
+            }
+          }
+          logger.info({ fromUrl: url, discovered: nextLinks.length, queueAfter: queue.length, visitedSoFar: visited.size }, '[crawler] link processing done');
+
+          // Harvest SPA routes captured via history.pushState/replaceState interception
+          try {
+            const jsRoutes = await page.evaluate(() => {
+              const routes = (window as any).__zetaDiscoveredRoutes ?? [];
+              (window as any).__zetaDiscoveredRoutes = [];
+              return routes as string[];
+            }).catch(() => [] as string[]);
+            for (const route of jsRoutes) {
+              try {
+                const abs = route.startsWith('http') ? route : `${origin}${route.startsWith('/') ? '' : '/'}${route}`;
+                const canonRoute = canonicalize(abs);
+                if (!queued.has(canonRoute) && !visited.has(canonRoute)) {
+                  queued.add(canonRoute);
+                  queue.push(abs);
+                  logger.info({ route: abs }, '[crawler] history.pushState: discovered SPA route');
+                }
+              } catch { /* ignore invalid routes */ }
+            }
+          } catch { /* non-fatal */ }
+
+          // P3a: Post-extraction DOM count — detect async-loaded content.
+          // Only check sparse pages (preDomCount < 25) where async widgets are likely.
+          // If DOM grew significantly during AI extraction, re-visit later with networkidle.
+          if (preDomCount < 25 && !revisitQueue.includes(url)) {
+            try {
+              const postDomCount = await page.evaluate(() =>
+                document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
+              ).catch(() => 0);
+              if (postDomCount > preDomCount * 1.3 + 4) {
+                revisitQueue.push(url);
+                logger.info({ url, preDomCount, postDomCount }, '[P3a] Async content detected — scheduled for re-visit');
+              }
+            } catch { /* non-fatal */ }
+          }
+
+          // P2: Static JS router detection — scan bundle for declared client-side routes.
+          // Only run on first 5 pages (bundle is the same across pages; scanning every page is wasteful).
+          if (visited.size <= 5) {
+            try {
+              const jsDetectedRoutes = await detectJsRoutes(page, origin);
+              for (const route of jsDetectedRoutes) {
+                const canonRoute = canonicalize(route);
+                if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 2) {
+                  queued.add(canonRoute);
+                  queue.push(route, { parentElements: elements.length });
+                  logger.info({ route }, '[P2] JS router: discovered route added to queue');
+                }
+              }
+            } catch { /* non-fatal */ }
+          }
+
+          // P4: Drain XHR-discovered page URL candidates (accumulated by network-observer callback).
+          // These are relative paths extracted from JSON API response bodies.
+          for (const relPath of [...xhrDiscoveredPaths]) {
+            xhrDiscoveredPaths.delete(relPath);
+            try {
+              const abs = `${origin}${relPath}`;
+              const canon = canonicalize(abs);
+              if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                queued.add(canon);
+                queue.push(abs);
+                logger.info({ url: abs }, '[P4] XHR: page URL from JSON response added to queue');
+              }
+            } catch { /* ignore invalid paths */ }
+          }
+
+          // Gap 5: Skyvern URL-discovery bootstrap for pages that defeat DOM link extraction.
+          // Triggered per-page when: zero DOM links found + fewer than 5 elements (dead-end / canvas page).
+          // Skyvern returns URLs it discovered → added to queue for ZeTa's own BFS crawl (not saved directly).
+          // ZeTa does all actual crawling — Skyvern is only a URL-discovery override layer.
+          if (nextLinks.length === 0 && elements.length < 5 && opts.skyvernConfig) {
+            try {
+              logger.info({ url }, 'Gap 5: thin page detected — asking Skyvern to discover navigation URLs');
+              const skyvernNav = await skyvernNavigate(
+                url,
+                ['Find and return all navigation links, menu items, and page URLs accessible from this screen'],
+                opts.skyvernConfig,
+              );
+              for (const skyvernUrl of skyvernNav.visitedUrls) {
+                const canonSkyvernUrl = canonicalize(skyvernUrl);
+                if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 2) {
+                  logger.info({ url: skyvernUrl }, 'Gap 5: Skyvern-discovered URL added to ZeTa crawl queue');
+                  queued.add(canonSkyvernUrl);
+                  queue.push(skyvernUrl); // ZeTa will crawl this normally
+                  // Wire the KAG edge so the Skyvern-discovered page isn't flagged as an orphan
+                  pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: 'Skyvern-discovered link', targetUrl: skyvernUrl });
+                }
+              }
+            } catch { /* non-fatal — Skyvern unavailable */ }
+          }
+        }
+
+        if (rawPerf) {
+          performanceMetrics.push({ url, screenName: name, ...rawPerf });
+        }
+
+        const screenApiEndpoints = capturedApiEndpoints.slice(apiObservationStartIndex);
+        if (screenApiEndpoints.length > 0) {
+          AppBrain.saveApiEndpointObservations(opts.tenantId, opts.projectId, screen.id, screenApiEndpoints)
+            .then(() => AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              screenId: screen.id,
+              jobId: opts.jobId,
+              type: 'API_ENDPOINT_OBSERVATIONS',
+              payload: { endpoints: screenApiEndpoints },
+            }))
+            .catch((err: any) => console.warn('[crawler] API endpoint observation save failed:', err?.message ?? err));
+        }
+
+        // Persist performance snapshot (web vitals + optional Lighthouse) — fire-and-forget
+        if (opts.jobId && (rawPerf || process.env.ENABLE_LIGHTHOUSE)) {
+          (async () => {
+            try {
+              const lhScores = await runLighthouse(url);
+              await AppBrain.savePerformanceSnapshot(opts.tenantId, screen.id, opts.jobId!, {
+                lcp: rawPerf?.lcp,
+                fcp: rawPerf?.fcp,
+                cls: rawPerf?.cls,
+                ttfb: rawPerf?.ttfb,
+                viewport: 'DESKTOP',
+                ...(lhScores ? {
+                  lighthousePerf: lhScores.perf,
+                  lighthouseA11y: lhScores.a11y,
+                  lighthouseSEO: lhScores.seo,
+                  lighthouseBP: lhScores.bp,
+                  lighthouseReport: lhScores.report,
+                } : {}),
+              });
+            } catch (perfErr) {
+              console.warn('[crawler] savePerformanceSnapshot failed (non-fatal):', perfErr);
+            }
+          })();
+        }
+
+        discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+        // Progress: 20% baseline + up to 60% across discovered screens.
+        // Use high-water mark (maxSeenTotal) so % never goes backwards when new URLs are found.
+        maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
+        const screenPct = Math.min(60, (visited.size / Math.max(1, maxSeenTotal)) * 60);
+        await onProgress?.(20 + Math.round(screenPct), `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
+        logger.info({ name, elements: elements.length }, 'Screen crawled');
+        // Checkpoint every 10 successes so a failed job can be resumed from close to where it stopped
+        if (++checkpointSuccessCount % 10 === 0) {
+          await opts.onCheckpoint?.(
+            Array.from(visited.keys()),
+            queue.filter(u => !visited.has(canonicalize(u)))
+          );
+        }
+      } catch (err: any) {
+        logger.warn({ url, err: err?.message ?? String(err), stack: err?.stack?.split('\n').slice(0, 5).join(' | ') }, 'Screen crawl failed');
+        discovered.push({ url, error: err?.message ?? String(err) });
+      }
+    }
+
+    // Discovery mode: return collected URLs and skip all post-crawl analysis phases.
+    // Use visited.values() (original URLs) not .keys() (canonical forms with /{id} placeholders).
+    if (opts.discoveryMode) {
+      return {
+        discoveredUrls: Array.from(visited.values()),
+        screensFound: 0,
+        screensErrored: 0,
+        screensSkippedToNextSegment: 0,
+        isPartial: false,
+        discovered: [],
+        apiEndpoints: [],
+        performanceMetrics: [],
+        pendingUrls: [],
+        segmentIndex: segIdx,
+        hasMoreSegments: false,
+        domHashes: Array.from(seenDomHashes),
+      };
+    }
+
+    // Gap 5: Skyvern post-crawl top-up for canvas/non-DOM apps where total screens < 10.
+    // Skyvern discovers URLs → ZeTa's own BFS crawl processes each one normally.
+    // Skyvern is ONLY a URL-discovery layer here — it does NOT save screens directly.
+    // Use only successfully-saved screens (screenId present), not total discovered including errors.
+    const successfulScreenCount = discovered.filter(d => d.screenId).length;
+    if (successfulScreenCount < 10 && opts.skyvernConfig) {
+      await onProgress?.(78, 'Few screens found — using Skyvern to discover additional navigation URLs');
+      const currentUrl = page.url();
+      const skyvernNav = await skyvernNavigate(
+        currentUrl,
+        [
+          'Explore the main navigation and visit each top-level section',
+          'Click any sidebar or top menu items to discover sub-pages',
+        ],
+        opts.skyvernConfig,
+      );
+      const skyvernUrls = skyvernNav.visitedUrls.filter((u) => !visited.has(canonicalize(u)));
+      if (skyvernUrls.length > 0) {
+        logger.info({ count: skyvernUrls.length }, 'Gap 5: Skyvern found new URLs — processing with ZeTa native crawler');
+        // Process each URL with ZeTa's own Playwright-based extraction
+        for (const skyvernUrl of skyvernUrls) {
+          if (visited.size >= maxScreens) break;
+          try {
+            await page.goto(skyvernUrl, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            await page.waitForTimeout(1500);
+            const skyvernName = new URL(skyvernUrl).pathname.split('/').filter(Boolean).pop() ?? 'Screen';
+            const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, skyvernName, skyvernUrl, opts.jobId, 'SKYVERN_AI');
+            const shotPath = `${opts.screenshotDir ?? '/tmp'}/${screen.id}-v${screen.version}.png`;
+            await waitForScreenshotReady(page);
+            await page.screenshot({ path: shotPath, fullPage: true });
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP');
+            const [elems, links, linkMappings] = await Promise.all([
+              extractElements(stagehand, page, elementLLM ?? undefined),
+              discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0),
+              extractLinkMappings(page, origin),
+            ]);
+            for (const el of elems) {
+              await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
+            }
+            visited.set(canonicalize(skyvernUrl), screen.id);
+            discovered.push({ url: skyvernUrl, screenId: screen.id, elements: elems.length, version: screen.version });
+            // Wire KAG element->screen transitions for this Skyvern-discovered page too —
+            // without this, any page only ever reached via Skyvern always looks orphaned
+            // (zero inbound edges) even when a real DOM link exists elsewhere pointing to it.
+            for (const lm of linkMappings) {
+              pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
+            }
+            // Also enqueue links Skyvern page found for further ZeTa crawling
+            for (const href of links) {
+              const c = canonicalize(href);
+              if (isUsefulNavigationUrl(href, origin) && !queued.has(c) && !visited.has(c)) { queued.add(c); queue.push(href); }
+            }
+          } catch (err) {
+            logger.warn({ url: skyvernUrl, err: String(err) }, 'Gap 5: ZeTa crawl of Skyvern URL failed');
+          }
+        }
+        // Drain any newly queued URLs from Skyvern pages
+        while (queue.length > 0 && visited.size < maxScreens) {
+          const href = queue.shift()!;
+          const canonHref = canonicalize(href);
+          if (visited.has(canonHref)) continue;
+          if (!isUsefulNavigationUrl(href, origin)) continue;
+          try {
+            await page.goto(href, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            await page.waitForTimeout(1000);
+            const nm = new URL(href).pathname.split('/').filter(Boolean).pop() ?? 'Screen';
+            const sc = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, 'SKYVERN_AI');
+            const sp = `${opts.screenshotDir ?? '/tmp'}/${sc.id}-v${sc.version}.png`;
+            await waitForScreenshotReady(page);
+            await page.screenshot({ path: sp, fullPage: true });
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, 'DESKTOP');
+            const [elems2, linkMappings2] = await Promise.all([
+              extractElements(stagehand, page, elementLLM ?? undefined),
+              extractLinkMappings(page, origin),
+            ]);
+            for (const el of elems2) {
+              await AppBrain.saveElement(opts.tenantId, opts.projectId, sc.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
+            }
+            visited.set(canonHref, sc.id);
+            discovered.push({ url: href, screenId: sc.id, elements: elems2.length, version: sc.version });
+            for (const lm of linkMappings2) {
+              pendingTransitions.push({ fromScreenId: sc.id, elementMeaning: lm.label, targetUrl: lm.href });
+            }
+          } catch { /* non-fatal */ }
+        }
+      }
+    }
+
+    // P1: URL Pattern LLM Completion — post-crawl LLM prediction of undiscovered URLs.
+    // Single LLM call predicts missing routes from discovered URL patterns, HEAD-validates each.
+    // Saves URL_CANDIDATES evidence so the ActivityDrawer shows what was found.
+    try {
+      const { runUrlPatternCompletion } = await import('./url-pattern-completer.js');
+      const llmCfgForPatterns = await getTaskAgentLLMConfig(opts.tenantId, 'crawl').catch(() => null);
+      const urlPatternCandidates = await runUrlPatternCompletion(
+        Array.from(visited.keys()),
+        opts.appUrl,
+        llmCfgForPatterns,
+      );
+      if (urlPatternCandidates.length > 0) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts.jobId,
+          type: 'URL_CANDIDATES',
+          payload: {
+            candidates: urlPatternCandidates,
+            hits: urlPatternCandidates.filter(c => c.status === 'hit').length,
+            total: urlPatternCandidates.length,
+          },
+        }).catch(() => { });
+        logger.info(
+          { hits: urlPatternCandidates.filter(c => c.status === 'hit').length, total: urlPatternCandidates.length },
+          '[P1] URL pattern completion saved to evidence',
+        );
+      }
+    } catch (err) {
+      logger.warn({ err: String(err) }, '[P1] URL pattern completion failed — non-fatal, continuing');
+    }
+
+    // P3a: Async content re-visit — revisit pages where DOM grew significantly during extraction.
+    // Uses networkidle wait + longer stability period to capture fully-loaded dashboard content.
+    if (revisitQueue.length > 0) {
+      await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content…`);
+      logger.info({ count: revisitQueue.length }, '[P3a] Starting async content re-visit pass');
+      for (const revisitUrl of revisitQueue.slice(0, 10)) { // cap at 10 re-visits
+        const existingScreenId = visited.get(canonicalize(revisitUrl));
+        if (!existingScreenId) continue;
+        try {
+          await page.goto(revisitUrl, { waitUntil: 'networkidle', timeout: 25_000 });
+          await page.waitForTimeout(2_000);
+          const shotPath2 = `${opts.screenshotDir ?? '/tmp'}/${existingScreenId}-revisit.png`;
+          await waitForScreenshotReady(page);
+          await page.screenshot({ path: shotPath2, fullPage: true });
+          // Update screenshot with the more complete render
+          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, 'DESKTOP').catch(() => { });
+          // Re-extract elements with the async-loaded content now visible
+          const revisitElements = await extractElements(stagehand, page, elementLLM ?? undefined).catch(() => []);
+          for (const el of revisitElements) {
+            await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
+          }
+          // Harvest any new links revealed by async content
+          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false).catch(() => []);
+          for (const href of revisitLinks) {
+            const canon = canonicalize(href);
+            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+              queued.add(canon);
+              queue.push(href);
+            }
+          }
+          logger.info({ url: revisitUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, '[P3a] Re-visit complete');
+        } catch (err) {
+          logger.warn({ url: revisitUrl, err: String(err) }, '[P3a] Re-visit failed — non-fatal');
+        }
+      }
+    }
+
+    // P6: Gap Pass — LLM gap analysis on discovered screens to find structurally missing pages.
+    // Runs after P3a re-visits so the gap analysis has the most complete screen list possible.
+    // Confirmed gap URLs (HEAD=200) are visited directly with the existing browser.
+    if (visited.size >= 3) {
+      try {
+        await onProgress?.(80, 'Running gap analysis…');
+        const llmCfgForGap = await getTaskAgentLLMConfig(opts.tenantId, 'crawl').catch(() => null);
+        const discoveredForGap = Array.from(visited.entries()).map(([url, screenId]) => {
+          const d = discovered.find(dd => dd.screenId === screenId);
+          return { url, name: d ? (new URL(url).pathname.split('/').filter(Boolean).pop() ?? 'screen') : undefined, elements: d?.elements };
+        });
+        const gapCandidates = await runGapAnalysis(discoveredForGap, opts.appUrl, llmCfgForGap);
+        const confirmedGaps = gapCandidates.filter(g => g.confirmedUrl && !visited.has(canonicalize(g.confirmedUrl!)));
+
+        if (confirmedGaps.length > 0) {
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: 'URL_CANDIDATES',
+            payload: {
+              source: 'gap-pass',
+              candidates: gapCandidates.map(g => ({ url: g.confirmedUrl ?? g.urlPatterns[0], confidence: g.confidence, reason: g.rationale, status: g.confirmedUrl ? 'hit' : 'miss' })),
+              hits: confirmedGaps.length,
+              total: gapCandidates.length,
+            },
+          }).catch(() => { });
+          logger.info({ confirmed: confirmedGaps.length, total: gapCandidates.length }, '[P6] Gap pass: visiting confirmed gap URLs');
+
+          // Visit each confirmed gap URL with the existing browser (depth=1, no link traversal)
+          const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
+          for (const gap of confirmedGaps.slice(0, 8)) {
+            const gapUrl = gap.confirmedUrl!;
+            if (visited.size >= maxScreens) break;
+            try {
+              await page.goto(gapUrl, { waitUntil: 'domcontentloaded', timeout: 20_000 });
+              // Skip if session expired — don't save a login-page screenshot as the gap result
+              if (gapLoginPattern.test(page.url())) {
+                logger.warn({ gapUrl, landed: page.url() }, '[P6] Gap URL redirected to login — skipping');
+                continue;
+              }
+              await page.waitForTimeout(1_500);
+              const gapName = gap.category;
+              const gapScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, gapName, gapUrl, opts.jobId, 'ENTRY_POINT');
+              const gapShot = `${opts.screenshotDir ?? '/tmp'}/${gapScreen.id}-v${gapScreen.version}.png`;
+              await waitForScreenshotReady(page);
+              await page.screenshot({ path: gapShot, fullPage: true });
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, 'DESKTOP');
+              const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
+                extractElements(stagehand, page, elementLLM ?? undefined).catch(() => []),
+                discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false).catch(() => []),
+                extractLinkMappings(page, origin).catch(() => []),
+              ]);
+              for (const el of gapElems) {
+                await AppBrain.saveElement(opts.tenantId, opts.projectId, gapScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
+              }
+              visited.set(canonicalize(gapUrl), gapScreen.id);
+              discovered.push({ url: gapUrl, screenId: gapScreen.id, elements: gapElems.length, version: gapScreen.version });
+              for (const lm of gapLinkMaps) {
+                pendingTransitions.push({ fromScreenId: gapScreen.id, elementMeaning: lm.label, targetUrl: lm.href });
+              }
+              // Enqueue gap page's links for further crawl (within maxScreens budget)
+              for (const href of gapLinks) {
+                const canon = canonicalize(href);
+                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                  queued.add(canon); queue.push(href);
+                }
+              }
+              logger.info({ url: gapUrl, elements: gapElems.length, category: gap.category }, '[P6] Gap URL captured');
+            } catch (err) {
+              logger.warn({ url: gapUrl, err: String(err) }, '[P6] Gap URL visit failed — non-fatal');
+            }
+          }
+        } else {
+          logger.info({ total: gapCandidates.length }, '[P6] Gap pass: no confirmed gaps found');
+        }
+      } catch (err) {
+        logger.warn({ err: String(err) }, '[P6] Gap pass failed — non-fatal, continuing');
+      }
+    }
+
+    // After crawl: mark screens not re-discovered as PENDING_REMOVAL (non-destructive diff)
+    // Only run if crawl actually found screens — 0 discovered means failed/aborted run, not evidence of removal
+    if (!selective && visited.size > 0) {
+      const discoveredUrls = Array.from(visited.keys());
+      await AppBrain.markRemovedScreens(opts.tenantId, opts.projectId, discoveredUrls, opts.jobId);
+    }
+
+    // Step 3: Save flows
+    await onProgress?.(82, 'Saving app map');
+    for (const edge of edges) {
+      const fromId = visited.get(canonicalize(edge.fromUrl));
+      const toId = visited.get(canonicalize(edge.toUrl));
+      if (fromId && toId && fromId !== toId) {
+        await AppBrain.saveFlow(opts.tenantId, opts.projectId, `${edge.fromUrl} -> ${edge.toUrl}`, fromId, toId);
+      }
+    }
+
+    // Step 4: Wire KAG element-level transitions
+    // For each <a href> link captured from each screen, resolve the href to a visited screenId
+    // and store transitionsToScreenId on the matching element.
+    await onProgress?.(84, 'Wiring navigation graph (KAG)');
+    // Secondary index: pathname → [canonicalUrl, ...] for subset-param fuzzy matching.
+    // A link to /product?id=5 should resolve to a page visited at /product?id=5&session=abc
+    // even though their canonical forms differ (session param not in the strip list).
+    const visitedByPath = new Map<string, string[]>();
+    for (const [canonUrl] of visited) {
+      try {
+        const p = new URL(canonUrl).pathname;
+        const arr = visitedByPath.get(p);
+        if (arr) arr.push(canonUrl); else visitedByPath.set(p, [canonUrl]);
+      } catch { /* invalid canonical — skip */ }
+    }
+    for (const pt of pendingTransitions) {
+      let toScreenId = visited.get(canonicalize(pt.targetUrl));
+      if (!toScreenId) {
+        // Fuzzy fallback: find a visited URL at the same pathname whose params are a superset
+        // of the link's params (handles extra session/tracking params on the visited side).
+        try {
+          const targetCanon = new URL(canonicalize(pt.targetUrl));
+          const candidates = visitedByPath.get(targetCanon.pathname) ?? [];
+          for (const cand of candidates) {
+            const candUrl = new URL(cand);
+            let allMatch = true;
+            for (const [k, v] of targetCanon.searchParams) {
+              if (candUrl.searchParams.get(k) !== v) { allMatch = false; break; }
+            }
+            if (allMatch) { toScreenId = visited.get(cand); break; }
+          }
+        } catch { /* non-fatal */ }
+      }
+      if (toScreenId && toScreenId !== pt.fromScreenId) {
+        await AppBrain.setElementTransition(opts.tenantId, pt.fromScreenId, pt.elementMeaning, toScreenId).catch(() => { });
+        await AppBrain.saveScreenTransition(opts.tenantId, opts.projectId, {
+          fromScreenId: pt.fromScreenId,
+          toScreenId,
+          action: {
+            type: 'click',
+            label: pt.elementMeaning,
+            targetUrl: pt.targetUrl,
+            source: 'crawl-link-mapping',
+          },
+          confidence: 0.8,
+          jobId: opts.jobId,
+        }).catch(() => { });
+      }
+    }
+    const transitionCount = pendingTransitions.filter(pt => {
+      if (visited.get(canonicalize(pt.targetUrl))) return true;
+      try {
+        const tc = new URL(canonicalize(pt.targetUrl));
+        return (visitedByPath.get(tc.pathname) ?? []).some(cand => {
+          const cu = new URL(cand);
+          for (const [k, v] of tc.searchParams) { if (cu.searchParams.get(k) !== v) return false; }
+          return true;
+        });
+      } catch { return false; }
+    }).length;
+    logger.info({ transitionCount }, 'KAG: wired element→screen transitions');
+    AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+      jobId: opts.jobId,
+      type: 'WORKFLOW_GRAPH_SUMMARY',
+      payload: {
+        transitionCount,
+        screenCount: visited.size,
+        profile: domainProfile.industry,
+      },
+    }).catch(() => { });
+
+    if (capturedApiEndpoints.length > 0) {
+      logger.info({ count: capturedApiEndpoints.length }, 'Captured unique API endpoint patterns');
+    }
+
+    if (visited.size >= 5) {
+      await onProgress?.(90, `App map saved · ${visited.size} screens, ${edges.length} flows wired`);
+    }
+
+    // P5: Orphan page auth re-verify — confirm session is still alive before tail-end viewport pass.
+    // The main BFS can span many minutes; the session may have expired by the time we reach here.
+    // If expired, attempt re-auth via Stagehand before proceeding. Only skip if re-auth also fails.
+    let sessionAliveForViewports = true;
+    try {
+      await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 });
+      const sessionCheckUrl = page.url();
+      const loginKeywords = /login|signin|sign-in|\/auth[/?#]/i;
+      if (loginKeywords.test(sessionCheckUrl)) {
+        logger.warn({ landed: sessionCheckUrl }, '[P5] Session expired before viewport pass — attempting re-auth');
+        let reauthed = false;
+        if (opts.credentials) {
+          reauthed = await loginWithStagehand(
+            stagehand, opts.credentials, opts.appUrl,
+            { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider },
+            { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId },
+            undefined,
+            { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId },
+          ).catch(() => false);
+        }
+        if (reauthed) {
+          logger.info('[P5] Re-auth succeeded — proceeding with viewport pass');
+        } else {
+          sessionAliveForViewports = false;
+          logger.warn({ landed: sessionCheckUrl }, '[P5] Re-auth failed — skipping mobile/tablet screenshots');
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: 'AUTH_FAILURE',
+            payload: {
+              url: opts.appUrl,
+              landed: sessionCheckUrl,
+              reason: 'Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings.',
+            },
+          }).catch(() => { });
+        }
+      } else {
+        logger.info({ url: sessionCheckUrl }, '[P5] Session alive — proceeding with viewport pass');
+      }
+    } catch (err) {
+      logger.warn({ err: String(err) }, '[P5] Session health check failed — skipping viewport pass to avoid bad screenshots');
+      sessionAliveForViewports = false;
+    }
+
+    // Step 5: Multi-viewport pass — screenshot + DOM element extraction at each device profile.
+    // Only DESKTOP (the main crawl above) runs Stagehand AI navigation and link discovery.
+    // Mobile/tablet viewports navigate to already-discovered screens and re-extract elements
+    // with their viewport-specific bounding rects.
+    //
+    // IMPORTANT: We launch a fresh Playwright browser per viewport profile instead of calling
+    // playwrightBrowser.newContext() on the CDP-connected instance. CDP-connected browsers
+    // share the DevTools session with Stagehand and cannot reliably set device emulation
+    // (isMobile, hasTouch, viewport, deviceScaleFactor) on new contexts — the flags are
+    // silently ignored, so screenshots come out at desktop dimensions regardless of profile.
+    const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== 'DESKTOP');
+    if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
+      const { devices, chromium: pwChromium } = await import('playwright');
+      const DEVICE_MAP: Record<string, any> = {
+        // Override viewport heights: Playwright's device descriptors use viewport-minus-browser-chrome
+        // (659 for iPhone 15, ~980 for iPad Pro 11). Override to full logical screen dimensions so
+        // screenshots match what users see on device and mobile-detection scripts pass.
+        MOBILE_PORTRAIT: { ...devices['iPhone 15'], viewport: { width: 393, height: 852 } },
+        TABLET: { ...devices['iPad Pro 11'], viewport: { width: 834, height: 1194 } },
+      };
+      // Export session cookies from the main crawl so mobile browser is already authenticated
+      let mainStorageState: any;
+      try {
+        mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState();
+      } catch { /* non-fatal — mobile pass will proceed without auth */ }
+
+      for (const profile of extraProfiles) {
+        const deviceConfig = DEVICE_MAP[profile];
+        if (!deviceConfig) continue;
+        await onProgress?.(95, `Capturing ${profile} screenshots + elements…`);
+
+        // Launch a dedicated browser for this viewport — no CDP sharing
+        let mobileBrowser: import('playwright').Browser | null = null;
+        try {
+          mobileBrowser = await pwChromium.launch({
+            headless: true,
+            executablePath,
+            args: chromeArgs,
+          });
+          const mobileCtx = await mobileBrowser.newContext({
+            ...deviceConfig,
+            ignoreHTTPSErrors: true,
+            ...(mainStorageState ? { storageState: mainStorageState } : {}),
+          });
+          const mobilePage = await mobileCtx.newPage();
+
+          for (const disc of discovered) {
+            if (!disc.screenId) continue;
+            try {
+              await mobilePage.goto(disc.url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
+              await mobilePage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
+              await dismissOverlays(mobilePage);
+              try { await mobilePage.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' }); } catch { /* ignore */ }
+              const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${disc.screenId}-${profile.toLowerCase()}-v${disc.version}.png`);
+              await waitForScreenshotReady(mobilePage);
+              // Screenshot and DOM element extraction run in parallel
+              const [, vpElements] = await Promise.all([
+                mobilePage.screenshot({ path: shotPath, fullPage: true }),
+                domFallbackElements(mobilePage),
+              ]);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version!, profile);
+              AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, disc.screenId, profile, shotPath, gitInfo.branch, gitInfo.commit || undefined).catch(() => { });
+              // Save viewport-specific element records (bounding rects differ per viewport)
+              for (const el of vpElements) {
+                AppBrain.saveElement(
+                  opts.tenantId, opts.projectId, disc.screenId,
+                  el.meaning, el.role, el.expectedData, el.notes,
+                  undefined, true, el.boundingRect, el.ariaState, el.parentLandmark,
+                  profile,
+                ).catch(() => { });
+              }
+            } catch (err) {
+              console.warn(`[crawler] ${profile} pass failed for ${disc.url}:`, err);
+            }
+          }
+          await mobilePage.close().catch(() => { });
+          await mobileCtx.close().catch(() => { });
+        } catch (profileErr) {
+          console.warn(`[crawler] ${profile} browser launch failed:`, profileErr);
+        } finally {
+          await mobileBrowser?.close().catch(() => { });
+        }
+      }
+    }
+
+    // Step 6: Interactive probe — observe real dynamic behaviour (optional, ~30s extra)
+    // Auto-enable for sparse crawls (< 30 screens): interaction-gated states are proportionally more common.
+    const shouldRunInteractiveProbe = opts.interactive || discovered.filter(d => d.screenId).length < 30;
+    if (shouldRunInteractiveProbe) {
+      await onProgress?.(88, 'Running interactive probe (capturing real errors & modals)');
+      const probeInputs = discovered
+        .filter((d) => d.screenId)
+        .map((d) => ({ url: d.url, screenId: d.screenId!, elements: d.elementFacts ?? [] }));
+      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
+    }
+
+    await onProgress?.(97, 'Running post-crawl analysis');
+
+    // ── Per-crawl summaries (fire-and-forget, non-blocking) ────────────────
+    const successScreens = discovered.filter(d => !!d.screenId);
+    ; (async () => {
+      try {
+        // 1. Sitemap XML
+        if (successScreens.length > 0) {
+          const sitemapUrls = screensToSitemapUrls(successScreens.map(d => ({ url: d.url })));
+          const sitemapPath = path.join(opts.screenshotDir ?? '/tmp', 'sitemap.xml');
+          await writeSitemap(sitemapUrls, sitemapPath);
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId, type: 'SITEMAP_WRITTEN', payload: { path: sitemapPath, urlCount: sitemapUrls.length },
+          }).catch(() => { });
+        }
+
+        // 2. OpenAPI spec from captured API endpoints
+        if (capturedApiEndpoints.length > 0) {
+          const spec = generateOpenApiSpec(capturedApiEndpoints, { title: `${opts.projectId} API`, version: '1.0.0' });
+          const yamlStr = specToYaml(spec);
+          const specPath = path.join(opts.screenshotDir ?? '/tmp', 'openapi.yaml');
+          await fs.writeFile(specPath, yamlStr, 'utf8');
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId, type: 'OPENAPI_SPEC', payload: { path: specPath, endpointCount: capturedApiEndpoints.length },
+          }).catch(() => { });
+        }
+
+        // 3. Dead link summary
+        const deadStats = deadLinkTracker.stats();
+        if (deadStats.total > 0) {
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId, type: 'DEAD_LINKS', payload: { ...deadStats, links: deadLinkTracker.getDeadLinks() },
+          }).catch(() => { });
+        }
+
+        // 4. Redirect chain summary
+        const redirectStats = redirectRegistry.stats();
+        if (redirectStats.withRedirects > 0) {
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId, type: 'REDIRECT_CHAINS', payload: { ...redirectStats, problematic: redirectRegistry.getProblematic() },
+          }).catch(() => { });
+        }
+
+        // 5. Playwright test generation (first 5 screens as sample)
+        if (successScreens.length > 0) {
+          const sampleScreens = successScreens.slice(0, 5).map(d => ({
+            id: d.screenId!, url: d.url, name: d.url,
+          }));
+          const playwrightScript = generatePlaywrightTest(sampleScreens);
+          const testPath = path.join(opts.screenshotDir ?? '/tmp', 'generated.spec.ts');
+          await fs.writeFile(testPath, playwrightScript, 'utf8');
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId, type: 'PLAYWRIGHT_TEST_GENERATED', payload: { path: testPath, screenCount: sampleScreens.length },
+          }).catch(() => { });
+        }
+
+        // 6. Webhook notification (if configured on project)
+        const webhookUrl = (opts as any).webhookUrl;
+        if (webhookUrl) {
+          const notifier = new WebhookNotifier({ url: webhookUrl, secret: (opts as any).webhookSecret });
+          await notifier.notify('crawl.completed', opts.jobId ?? '', opts.projectId, opts.tenantId, {
+            screensFound: successScreens.length,
+            apiEndpoints: capturedApiEndpoints.length,
+            deadLinks: deadLinkTracker.stats().total,
+          }).catch(() => { });
+        }
+      } catch { /* post-crawl analysis is non-fatal */ }
+    })();
+
+    await onProgress?.(98, 'Finalizing');
+    // Remaining queue = URLs discovered but not yet crawled due to segment limit
+    const pendingUrls = queue.filter((u) => !visited.has(canonicalize(u)));
+    const hasMoreSegments = pendingUrls.length > 0;
+    const screensSucceeded = discovered.filter(d => !!d.screenId).length;
+    const screensErrored = discovered.filter(d => !!d.error && !d.screenId).length;
+
+    return {
+      screensFound: screensSucceeded,           // successful captures only (not errors)
+      screensErrored,                           // pages visited but failed to capture
+      screensSkippedToNextSegment: pendingUrls.length,
+      isPartial: hasMoreSegments,               // true = more segments auto-queued
+      discovered,
+      apiEndpoints: capturedApiEndpoints,
+      performanceMetrics,
+      pendingUrls,                              // feed into next segment's resumeUrls
+      segmentIndex: segIdx,                     // which segment this was
+      hasMoreSegments,
+      domHashes: Array.from(seenDomHashes),     // carry forward to next segment to avoid re-screenshotting
+    };
+  } finally {
+    // Log accumulated Stagehand LLM usage (extract/act/observe calls) to cost ledger.
+    if (stagehandPromptTokens > 0 || stagehandCompletionTokens > 0) {
+      const totalTokens = stagehandPromptTokens + stagehandCompletionTokens;
+      const shProvider = stagehandModel.includes('/') ? stagehandModel.split('/')[0] : configuredProvider || 'openai';
+      const shModel = stagehandModel.includes('/') ? stagehandModel.split('/').slice(1).join('/') : stagehandModel;
+      const shUsage = {
+        provider: shProvider,
+        model: shModel,
+        promptTokens: stagehandPromptTokens,
+        completionTokens: stagehandCompletionTokens,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        totalTokens,
+      };
+      recordLLMCall({
+        tenantId: opts.tenantId,
+        projectId: opts.projectId,
+        jobId: opts.jobId,
+        agentType: 'crawl_stagehand',
+        label: 'stagehand_browser_actions',
+        usage: { ...shUsage, estimatedCostUsd: estimateCost(shUsage, `${shProvider}/${shModel}`) },
+      }).catch(() => { });
+    }
+    await playwrightBrowser?.close().catch(() => { });
+    await stagehand.close().catch(() => { });
+    await chromeProc.close().catch(() => { });
+  }
+}
+
+// Elements whose meanings indicate UI state toggles or noise — filtered from both DOM and AI results.
+const ELEMENT_NOISE = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
+
+async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
+  // llmCfg reserved for per-task model selection (future: create task-scoped Stagehand)
+  // DOM-first: fast, no LLM cost, covers server-rendered and most SPAs.
+  // Threshold >= 2 so a lone dark-mode button doesn't suppress AI fallback for rich pages.
+  const domElements = await domFallbackElements(page);
+  if (domElements.length >= 2) return domElements;
+
+  // Content fallback: static/marketing pages have no interactive widgets but have headings,
+  // images, links, and meta tags that feed SEO/a11y/visual test generation.
+  const contentElements = await contentFallbackElements(page);
+  if (contentElements.length > 0) return [...domElements, ...contentElements];
+
+  // AI fallback: DOM found nothing or only noise — try AI (pure SPA, shadow DOM, canvas-based).
+  try {
+    const extracted = await Promise.race([
+      stagehand.extract(
+        'Extract ONLY the meaningful form and action elements on this page — inputs, selects, textareas, and action buttons. ' +
+        'For each: a clear human-readable purpose (e.g. "Email address input", "Submit password reset button"), ' +
+        'its role (button/input/select/checkbox/textarea), and for inputs the expected data type ' +
+        '(valid_email/password/phone/date/search_term/text). ' +
+        'SKIP: navigation links, header/footer links, cookie banners, theme toggles (dark mode/light mode), ' +
+        'social share buttons, version strings, decorative elements, any button inside a nav or header. ' +
+        'Focus ONLY on main content forms and primary action buttons. Return at most 20 elements.',
+        ElementSchema as any
+      ),
+      new Promise<null>((resolve) => setTimeout(() => resolve(null), 30_000)),
+    ]);
+    if (!extracted) return domElements;
+    // Apply noise filter to AI results — AI doesn't always follow skip instructions
+    const filtered = (extracted as any).elements.filter((e: any) => !ELEMENT_NOISE.test(e.meaning ?? ''));
+    return filtered.slice(0, 20);
+  } catch {
+    return domElements;
+  }
+}
+
+async function discoverLinks(stagehand: Stagehand, origin: string, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, skipAi = false): Promise<string[]> {
+  // Expand collapsed nav menus (hamburger, aria-haspopup, Radix closed triggers) so hidden routes
+  // become part of the DOM before querying. This catches SPA nav dropdowns that only render on click.
+  const navExpanded = await expandNavMenus(page);
+
+  // DOM-first: <a href> links — standard apps. AI supplements when DOM coverage is < 20 unique links.
+  // Threshold of 20 (not 8) because nav sidebars quickly provide 8+ links while missing deep content URLs.
+  const domLinks = await domFallbackLinks(page, origin);
+
+  // Interact with tabs, carousels, and accordions to reveal hidden content and discover more links.
+  const interactiveLinks = await interactAndCollectLinks(page, origin);
+
+  // Dismiss any opened menus so page is clean for screenshot/element extraction after this call.
+  if (navExpanded) {
+    page.keyboard?.press('Escape').catch(() => { });
+    await page.evaluate(() => {
+      // Also click any currently-open triggers to close them (toggle pattern)
+      document.querySelectorAll<HTMLElement>('[aria-expanded="true"],[data-state="open"],[data-headlessui-state="open"]').forEach(el => {
+        try { el.click(); } catch { /* ignore */ }
+      });
+    }).catch(() => { });
+  }
+
+  const allDomLinks = Array.from(new Set([...domLinks, ...interactiveLinks]));
+
+  if (allDomLinks.length >= 30 || skipAi) return allDomLinks;
+
+  // AI complement: DOM found fewer than 30 unique links — SPA likely has JS-router pages not in <a href>.
+  // Always merge DOM + AI so we never lose already-discovered links.
+  // 12s timeout: allows 1 OpenAI retry (4.8s delay) but bails before the 2nd retry wastes more time.
+  try {
+    const extracted = await Promise.race([
+      stagehand.extract(
+        `Find all in-app navigation links for this application (origin: ${origin}). ` +
+        'Include: main nav, sidebar, content links, feature pages, settings pages. ' +
+        'Exclude: logout, sign out, delete, external sites, anchor-only links (#), mailto:, javascript:.',
+        LinksSchema as any
+      ),
+      new Promise<null>((resolve) => setTimeout(() => resolve(null), 12_000)),
+    ]);
+    if (!extracted) return allDomLinks;
+    const { links } = extracted as any;
+    const currentUrl: string = page.url?.() ?? '';
+    const aiLinks = links
+      .map((href: string) => {
+        try { const u = new URL(href, currentUrl); if (!/^#!?\//.test(u.hash)) u.hash = ''; return u.toString(); }
+        catch { return null; }
+      })
+      .filter((u: string | null): u is string => {
+        if (!u) return false;
+        return isUsefulNavigationUrl(u, origin);
+      });
+    // Always merge DOM + AI — AI complements, never replaces
+    return Array.from(new Set([...allDomLinks, ...aiLinks])).filter((href) => isUsefulNavigationUrl(href, origin));
+  } catch (err: any) {
+    // Detect OpenAI rate-limit errors — signal caller via thrown special error so circuit breaker can trip
+    const is429 = err?.status === 429 || String(err?.message ?? '').includes('429') || String(err?.message ?? '').includes('rate limit');
+    if (is429) throw Object.assign(new Error('RATE_LIMITED'), { isRateLimit: true, domLinks: allDomLinks });
+    return allDomLinks;
+  }
+}
+
+async function domFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
+  return page.evaluate(() => {
+    const out: { meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }[] = [];
+    const seen = new Set<string>();
+    // UI chrome noise: cookie banners, theme toggles, version strings, social/share buttons
+    const noise = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
+
+    const querySelectorAllDeep = (root: any, selector: string): Element[] => {
+      const results: Element[] = Array.from(root.querySelectorAll(selector));
+      root.querySelectorAll('*').forEach((el: any) => {
+        if (el.shadowRoot) {
+          results.push(...querySelectorAllDeep(el.shadowRoot, selector));
+        }
+      });
+      return results;
+    };
+
+    // Expanded selector with more ARIA roles + rich text editors
+    const selector = 'input:not([type=hidden]),select,textarea,button,[role="button"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],a[role="button"],[contenteditable="true"]';
+
+    querySelectorAllDeep(document, selector).forEach((el: any) => {
+      const style = window.getComputedStyle(el);
+      // Enhanced actionability checks
+      if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0' || style.pointerEvents === 'none') return;
+      if (el.hasAttribute('inert') || el.getAttribute('aria-disabled') === 'true') return;
+
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 && rect.height === 0) return;
+
+      const tag = el.tagName.toLowerCase();
+      const ariaRole = el.getAttribute('role');
+      const isActionEl = tag === 'button' || tag === 'a' || ariaRole === 'button';
+
+      // Skip plain anchor links inside <header> or <footer> — navigation chrome with no test value.
+      // Do NOT skip <nav> elements: SPA dashboard sidebars live in <nav> and are primary test targets.
+      // Buttons inside nav (e.g. collapse, settings) are kept; only <a> links are skipped.
+      if (tag === 'a' || (ariaRole === 'button' && tag !== 'button')) {
+        let p = el.parentElement;
+        while (p) {
+          const ptag = p.tagName?.toLowerCase();
+          if (['header', 'footer'].includes(ptag)) return;
+          p = p.parentElement;
+        }
+      }
+
+      // Label resolution: explicit <label for>, implicit wrapping <label>, aria-label, placeholder
+      let labelText = '';
+      if (el.id) {
+        const linkedLabel = document.querySelector(`label[for="${el.id}"]`);
+        if (linkedLabel) labelText = linkedLabel.textContent?.trim() || '';
+      }
+      // Implicit label: input is a child of <label> (no for= attribute needed)
+      if (!labelText) {
+        const wrappingLabel = el.closest('label');
+        if (wrappingLabel) {
+          const clone = wrappingLabel.cloneNode(true) as Element;
+          clone.querySelectorAll('input,select,textarea,button').forEach((c: any) => c.remove());
+          labelText = clone.textContent?.replace(/\s+/g, ' ').trim() || '';
+        }
+      }
+
+      const label = (
+        el.getAttribute('aria-label') ||
+        el.getAttribute('placeholder') ||
+        labelText ||
+        el.innerText?.replace(/\s+/g, ' ').trim() ||
+        el.getAttribute('name') ||
+        ''
+      ).slice(0, 80);
+
+      if (!label || noise.test(label) || label.length > 70) return;
+
+      const role = tag === 'input' ? (el.type || 'text') : el.hasAttribute('contenteditable') ? 'rich-text' : (ariaRole || tag);
+      const key = `${role}:${label.toLowerCase()}`;
+      if (seen.has(key)) return;
+      seen.add(key);
+
+      let expectedData: string | undefined;
+      if (tag === 'input') {
+        if (el.type === 'email' || /email/i.test(label)) expectedData = 'valid_email';
+        else if (el.type === 'password' || /password/i.test(label)) expectedData = 'password';
+        else if (el.type === 'tel' || /phone|mobile/i.test(label)) expectedData = 'phone';
+        else if (el.type === 'date' || /\bdate\b/i.test(label)) expectedData = 'date';
+        else if (/\b(code|otp|token)\b/i.test(label)) expectedData = 'one_time_code';
+        else if (/\b(search|query)\b/i.test(label)) expectedData = 'search_term';
+        else if (el.type === 'number' || /\b(amount|quantity|count|age|year)\b/i.test(label)) expectedData = 'number';
+        else if (el.type === 'url' || /\b(url|website|link)\b/i.test(label)) expectedData = 'url';
+        else if (el.type === 'file') expectedData = 'file';
+      }
+
+      // Multi-Attribute Identity
+      const attrs: Record<string, string> = {};
+      if (el.id) attrs.id = el.id;
+      if (el.name) attrs.name = el.name;
+      if (el.getAttribute('data-testid')) attrs['data-testid'] = el.getAttribute('data-testid');
+
+      // HTML5 validation attributes — ground test boundary values in real constraints
+      if (tag === 'input' || tag === 'textarea') {
+        if (el.required) attrs.required = 'true';
+        if (el.pattern) attrs.pattern = el.pattern;
+        if (el.minLength > 0) attrs.minLength = String(el.minLength);
+        if (el.maxLength > 0 && el.maxLength < 524288) attrs.maxLength = String(el.maxLength);
+        if (el.min !== '') attrs.min = String(el.min);
+        if (el.max !== '') attrs.max = String(el.max);
+        if (el.step && el.step !== 'any') attrs.step = String(el.step);
+        if (el.accept) attrs.accept = el.accept;
+        if (el.multiple) attrs.multiple = 'true';
+        if (el.readOnly) attrs.readOnly = 'true';
+        if (el.disabled) attrs.disabled = 'true';
+        if (el.autocomplete && el.autocomplete !== 'on') attrs.autocomplete = el.autocomplete;
+      }
+      // CSS class hint — first component-library class for COMPONENT file detection
+      const cssClass = (el.className || '')
+        .split(' ')
+        .map((c: string) => c.trim())
+        .find((c: string) => c.length > 2 && !/^(true|false|active|disabled|hidden|visible)$/i.test(c));
+      if (cssClass) attrs.cssClass = cssClass;
+
+      // GAP 4: Select option enumeration — test generator knows valid values
+      if (tag === 'select' && el.options?.length > 0 && el.options.length <= 60) {
+        const opts = Array.from(el.options).map((o: any) => {
+          const entry: Record<string, any> = { v: o.value, l: o.text.trim() };
+          if (o.selected) entry.s = true;
+          return entry;
+        }).filter((o: any) => o.l);
+        if (opts.length > 0) attrs.selectOptions = JSON.stringify(opts);
+      }
+
+      // GAP 6: aria-describedby help text — captures real constraint text (e.g. "min 8 chars")
+      const describedById = el.getAttribute('aria-describedby');
+      if (describedById) {
+        const helpEl = document.getElementById(describedById.split(' ')[0]);
+        const helpText = helpEl?.textContent?.trim().slice(0, 200);
+        if (helpText) attrs.helpText = helpText;
+      }
+
+      // GAP 1: Form grouping — inputs/buttons tagged with their containing form
+      const form = el.closest('form');
+      if (form) {
+        if ((form as any).id) attrs.formId = (form as any).id;
+        const formAction = form.getAttribute('action');
+        if (formAction && !formAction.startsWith('javascript:')) attrs.formAction = formAction.slice(0, 120);
+        const formMethod = (form as any).method;
+        if (formMethod) attrs.formMethod = formMethod.toUpperCase();
+      }
+      const fieldset = el.closest('fieldset');
+      if (fieldset) {
+        const legendText = fieldset.querySelector('legend')?.textContent?.trim().slice(0, 60);
+        if (legendText) attrs.fieldsetGroup = legendText;
+      }
+
+      // GAP 7: Button classification — type + destructive intent
+      if (tag === 'button' || ariaRole === 'button') {
+        if (el.type && el.type !== 'button') attrs.buttonType = el.type; // submit | reset
+        if (/\b(delete|remove|destroy|revoke|clear|reset|discard|cancel)\b/i.test(label)) attrs.isDestructive = 'true';
+      }
+
+      // ARIA state snapshot — promoted to first-class field (Phase 1)
+      const stateMap: Record<string, string> = {};
+      const ariaExpanded = el.getAttribute('aria-expanded');
+      const ariaChecked = el.getAttribute('aria-checked');
+      const ariaPressed = el.getAttribute('aria-pressed');
+      const ariaSelected = el.getAttribute('aria-selected');
+      const ariaInvalid = el.getAttribute('aria-invalid');
+      const ariaCurrent = el.getAttribute('aria-current');
+      if (ariaExpanded !== null) stateMap.expanded = ariaExpanded;
+      if (ariaChecked !== null) stateMap.checked = ariaChecked;
+      if (ariaPressed !== null) stateMap.pressed = ariaPressed;
+      if (ariaSelected !== null) stateMap.selected = ariaSelected;
+      if (ariaInvalid && ariaInvalid !== 'false') stateMap.invalid = ariaInvalid;
+      if (ariaCurrent) stateMap.current = ariaCurrent;
+      if (el.getAttribute('aria-required') === 'true' && !attrs.required) stateMap.required = 'true';
+
+      // Parent landmark — promoted to first-class field (Phase 1)
+      const getRegion = (node: any): string | undefined => {
+        let p = node.parentElement;
+        while (p && p !== document.body) {
+          const ptag = p.tagName?.toLowerCase();
+          const prole = p.getAttribute?.('role');
+          if (prole === 'dialog' || prole === 'alertdialog') return 'modal';
+          if (prole === 'main' || ptag === 'main') return 'main';
+          if (prole === 'complementary' || ptag === 'aside') return 'aside';
+          if (prole === 'navigation' || ptag === 'nav') return 'nav';
+          if (prole === 'banner' || ptag === 'header') return 'header';
+          if (prole === 'contentinfo' || ptag === 'footer') return 'footer';
+          p = p.parentElement;
+        }
+        return undefined;
+      };
+      const parentLandmark = getRegion(el);
+
+      // GAP 8: Rich text editor detection
+      if (el.hasAttribute('contenteditable')) {
+        const cls = ((el.className || '') + ' ' + (el.getAttribute('data-type') || '')).toLowerCase();
+        const editorType = cls.includes('tiptap') || cls.includes('prosemirror') ? 'tiptap' :
+          cls.includes('cm-') || cls.includes('codemirror') ? 'codemirror' :
+            (window as any).Quill ? 'quill' :
+              typeof (window as any).monaco !== 'undefined' ? 'monaco' : 'contenteditable';
+        attrs.editorType = editorType;
+      }
+
+      const notes = Object.keys(attrs).length > 0 ? JSON.stringify(attrs) : undefined;
+      const boundingRect = {
+        x: Math.round(rect.x), y: Math.round(rect.y),
+        width: Math.round(rect.width), height: Math.round(rect.height),
+        top: Math.round(rect.top), left: Math.round(rect.left),
+        right: Math.round(rect.right), bottom: Math.round(rect.bottom),
+      };
+      out.push({
+        meaning: label, role, expectedData, notes,
+        boundingRect,
+        ariaState: Object.keys(stateMap).length > 0 ? stateMap : undefined,
+        parentLandmark: parentLandmark ?? undefined,
+      });
+    });
+
+    // Form fields first, then buttons
+    const priority = (e: typeof out[0]) =>
+      ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea', 'url', 'file'].includes(e.role) ? 0 : 1;
+    out.sort((a, b) => priority(a) - priority(b));
+    return out; // Cap removed
+  }).catch(() => []);
+}
+
+/** For pages with no interactive elements (marketing/static pages), capture content anatomy:
+ *  headings, images-with-alt, CTA links outside nav/header/footer, and meta tags.
+ *  These give the test generator something to work with for SEO, a11y, and visual tests. */
+async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string }>> {
+  return page.evaluate(() => {
+    const out: { meaning: string; role: string; notes?: string }[] = [];
+    const seen = new Set<string>();
+    const addUniq = (item: { meaning: string; role: string; notes?: string }) => {
+      const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
+      if (!seen.has(key) && item.meaning.trim().length > 0) { seen.add(key); out.push(item); }
+    };
+
+    // 1. Page-level meta (title, description, og:)
+    const title = document.title?.trim();
+    if (title) addUniq({ meaning: title, role: 'page-title', notes: JSON.stringify({ tag: 'title' }) });
+    const desc = document.querySelector('meta[name="description"]')?.getAttribute('content')?.trim();
+    if (desc) addUniq({ meaning: desc, role: 'meta-description', notes: JSON.stringify({ tag: 'meta[name=description]' }) });
+    const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim();
+    if (ogTitle) addUniq({ meaning: ogTitle, role: 'og-title', notes: JSON.stringify({ tag: 'meta[property=og:title]' }) });
+    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href')?.trim();
+    if (canonical) addUniq({ meaning: canonical, role: 'canonical-url', notes: JSON.stringify({ tag: 'link[rel=canonical]' }) });
+
+    // 2. Headings — page structure
+    document.querySelectorAll('h1, h2, h3').forEach((h: any) => {
+      const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80);
+      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
+    });
+
+    // 3. Images with alt text — accessibility targets
+    document.querySelectorAll('img[alt]').forEach((img: any) => {
+      const alt = img.getAttribute('alt')?.trim().slice(0, 80);
+      const src = img.getAttribute('src')?.slice(0, 120);
+      if (alt) addUniq({ meaning: alt, role: 'image', notes: JSON.stringify({ src }) });
+    });
+
+    // 4. CTA / content links — outside nav, header, footer
+    const navTags = new Set(['nav', 'header', 'footer']);
+    document.querySelectorAll('a[href]').forEach((a: any) => {
+      let p = a.parentElement;
+      while (p) { if (navTags.has(p.tagName?.toLowerCase())) return; p = p.parentElement; }
+      const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 80);
+      const href = a.href;
+      if (label && href && !/^(#|javascript:|mailto:)/i.test(href)) {
+        const isExternal = !href.includes(location.hostname);
+        addUniq({ meaning: label, role: isExternal ? 'external-link' : 'internal-link', notes: JSON.stringify({ href: href.slice(0, 120) }) });
+      }
+    });
+
+    // 5. Animated / interactive content markers (CSS animation, video, canvas)
+    const hasAnimation = !!document.querySelector('[class*="animate"], [class*="motion"], [class*="transition"], [class*="scroll-"], [data-aos], [data-scroll]');
+    if (hasAnimation) addUniq({ meaning: 'CSS animations / scroll effects detected on page', role: 'animation-layer', notes: JSON.stringify({ detected: true }) });
+    if (document.querySelector('video')) addUniq({ meaning: 'Video element present', role: 'video', notes: JSON.stringify({ count: document.querySelectorAll('video').length }) });
+    if (document.querySelector('canvas')) addUniq({ meaning: 'Canvas element present', role: 'canvas' });
+
+    return out.slice(0, 30);
+  }).catch(() => []);
+}
+
+/**
+ * Click through tabs, advance carousels, open accordions — collect links that only appear
+ * inside those interactive panels. Does not navigate away from the current page.
+ */
+async function interactAndCollectLinks(page: any, origin: string): Promise<string[]> {
+  const collected = new Set<string>();
+  const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
+
+  const collectCurrentLinks = async () => {
+    const links: string[] = await page.evaluate((origin: string) => {
+      const out: string[] = [];
+      const logout = /logout|log out|sign out|delete/i;
+      document.querySelectorAll<HTMLAnchorElement>('a[href]').forEach(a => {
+        if (logout.test(a.textContent ?? '')) return;
+        try {
+          const u = new URL(a.href);
+          if (u.origin === origin) { if (u.hash && !/^#!?\//.test(u.hash)) u.hash = ''; out.push(u.toString()); }
+        } catch { /* ignore */ }
+      });
+      return out;
+    }, origin).catch(() => []);
+    links.forEach(l => collected.add(l));
+  };
+
+  try {
+    // ── Tabs: click each tab individually, collect links from revealed panel ──
+    const tabCount: number = await page.evaluate(() =>
+      document.querySelectorAll('[role="tab"]').length
+    ).catch(() => 0);
+
+    for (let i = 0; i < Math.min(tabCount, 15); i++) {
+      try {
+        const clicked = await page.evaluate((idx: number) => {
+          const tabs = document.querySelectorAll<HTMLElement>('[role="tab"]');
+          const tab = tabs[idx];
+          if (!tab) return false;
+          const s = window.getComputedStyle(tab);
+          const r = tab.getBoundingClientRect();
+          if (s.display === 'none' || r.width < 4 || r.height < 4) return false;
+          tab.click();
+          return true;
+        }, i).catch(() => false);
+        if (clicked) {
+          await sleep(350);
+          await collectCurrentLinks();
+        }
+      } catch { /* skip */ }
+    }
+
+    // ── Carousels: advance next button up to 8 times, collect links each slide ──
+    const carouselNextSel = [
+      'button[aria-label*="next" i]', 'button[aria-label*="forward" i]',
+      'button[aria-label*="right" i]', '[data-direction="next"]',
+      '[class*="carousel" i] button[class*="next" i]',
+      '[class*="slider" i] button[class*="next" i]',
+    ].join(', ');
+
+    for (let i = 0; i < 8; i++) {
+      const advanced: boolean = await page.evaluate((sel: string) => {
+        const btn = document.querySelector<HTMLElement>(sel);
+        if (!btn) return false;
+        const s = window.getComputedStyle(btn);
+        const r = btn.getBoundingClientRect();
+        if (s.display === 'none' || r.width < 4 || r.height < 4) return false;
+        btn.click();
+        return true;
+      }, carouselNextSel).catch(() => false);
+      if (!advanced) break;
+      await sleep(300);
+      await collectCurrentLinks();
+    }
+
+    // ── Accordions / disclosure panels: open collapsed sections ──────────────
+    const accordionCount: number = await page.evaluate(() =>
+      document.querySelectorAll('details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]').length
+    ).catch(() => 0);
+
+    for (let i = 0; i < Math.min(accordionCount, 10); i++) {
+      try {
+        await page.evaluate((idx: number) => {
+          const items = document.querySelectorAll<HTMLElement>(
+            'details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]'
+          );
+          const el = items[idx];
+          if (el) { try { el.click(); } catch { /* ignore */ } }
+        }, i).catch(() => { });
+        await sleep(250);
+        await collectCurrentLinks();
+      } catch { /* skip */ }
+    }
+  } catch { /* best-effort */ }
+
+  return Array.from(collected);
+}
+
+/** Click collapsed nav triggers (hamburger menus, aria-haspopup buttons, Radix/Headless UI toggles)
+ *  to reveal routes that only exist in the DOM after user interaction.
+ *  Returns true if anything was clicked so caller can dismiss afterward. */
+async function expandNavMenus(page: any): Promise<boolean> {
+  try {
+    const clicked = await page.evaluate(() => {
+      const isVisible = (el: HTMLElement) => {
+        const s = window.getComputedStyle(el);
+        if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;
+        const r = el.getBoundingClientRect();
+        return r.width > 4 && r.height > 4;
+      };
+      const inBadContext = (el: HTMLElement) => {
+        let p = el.parentElement;
+        while (p) {
+          const tag = p.tagName?.toLowerCase();
+          const role = p.getAttribute('role') ?? '';
+          if (['form', 'table', 'select'].includes(tag ?? '') ||
+            ['dialog', 'alertdialog', 'combobox', 'listbox', 'grid', 'treegrid'].includes(role)) return true;
+          p = p.parentElement;
+        }
+        return false;
+      };
+      const targets = new Set<HTMLElement>();
+      // aria-haspopup: canonical menu-trigger pattern
+      document.querySelectorAll<HTMLElement>('[aria-haspopup]:not([aria-expanded="true"])').forEach(el => {
+        if (isVisible(el) && !inBadContext(el)) targets.add(el);
+      });
+      // aria-expanded=false: collapsed nav sections
+      document.querySelectorAll<HTMLElement>('[aria-expanded="false"]').forEach(el => {
+        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadContext(el)) targets.add(el);
+      });
+      // Radix UI / Headless UI closed triggers
+      document.querySelectorAll<HTMLElement>('[data-state="closed"],[data-headlessui-state="closed"]').forEach(el => {
+        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadContext(el)) targets.add(el);
+      });
+      let count = 0;
+      targets.forEach(el => { try { (el as HTMLElement).click(); count++; } catch { /* ignore */ } });
+      return count;
+    });
+    if (clicked > 0) await new Promise(r => setTimeout(r, 500));
+
+    // ── Hover sweep: reveal CSS :hover dropdown menus ──────────────────────
+    // Pure CSS hover menus need real mouse moves — JS click alone won't trigger :hover
+    try {
+      const navHandles = await page.$$(
+        'nav > ul > li, nav > li, header > ul > li, header > nav > ul > li, ' +
+        '[role="navigation"] > ul > li, [role="menubar"] > [role="menuitem"], ' +
+        '[class*="nav"] > ul > li, [class*="menu"] > li'
+      ).catch(() => [] as any[]);
+      for (const h of (navHandles as any[]).slice(0, 20)) {
+        try { await h.hover({ force: true, timeout: 600 }); await page.waitForTimeout(280); } catch { /* skip invisible */ }
+      }
+      if ((navHandles as any[]).length > 0) {
+        await page.mouse.move(2, 2); // dismiss hover state
+        await page.waitForTimeout(150);
+      }
+    } catch { /* non-fatal */ }
+
+    return (clicked as number) > 0;
+  } catch { return false; }
+}
+
+async function domFallbackLinks(page: any, origin: string): Promise<string[]> {
+  return page.evaluate((origin: string) => {
+    const logout = /logout|log out|sign out|delete/i;
+    // Hash-routed SPAs (React Router hash history, Vue Router hash mode, old
+    // AngularJS) encode the actual route as the fragment — "#/dashboard",
+    // "#!/settings" — as distinct from a same-page scroll anchor like "#top"
+    // or "#section-2". Treating every hash as a scroll anchor (stripping it
+    // entirely) collapses an entire hash-routed app down to its single base
+    // URL, discovering only ~1 screen no matter how many routes it has.
+    const isRouteHash = (hash: string) => /^#!?\//.test(hash);
+    const seen = new Set<string>();
+    const add = (rawHref: string, textContent: string) => {
+      if (logout.test(textContent)) return;
+      if (!rawHref) return;
+      if (rawHref.startsWith('#') && !isRouteHash(rawHref)) return;
+      if (rawHref.startsWith('mailto:') || rawHref.startsWith('javascript:')) return;
+      try {
+        const u = new URL(rawHref, location.href);
+        if (!isRouteHash(u.hash)) u.hash = '';
+        if (new URL(u).origin === origin) seen.add(u.toString());
+      } catch { /* ignore */ }
+    };
+
+    // Iterative BFS shadow DOM traversal — avoids stack overflow on deeply nested shadow roots
+    const allRoots: Array<Document | ShadowRoot> = [document];
+    const rootSeen = new WeakSet<Document | ShadowRoot>();
+    rootSeen.add(document);
+    while (allRoots.length > 0) {
+      const root = allRoots.shift();
+      if (!root) continue;
+      try {
+        root.querySelectorAll('a[href]').forEach((a) => add((a as HTMLAnchorElement).href || a.getAttribute('href') || '', a.textContent || ''));
+        root.querySelectorAll('[href]:not(a):not(link):not(base):not(script)').forEach((el) => {
+          add(el.getAttribute('href') || '', el.textContent || '');
+        });
+        root.querySelectorAll('[data-href]').forEach((el) => {
+          add(el.getAttribute('data-href') || '', el.textContent || '');
+        });
+        root.querySelectorAll('*').forEach((el) => {
+          try {
+            if (el.shadowRoot && !rootSeen.has(el.shadowRoot)) {
+              rootSeen.add(el.shadowRoot);
+              allRoots.push(el.shadowRoot);
+            }
+          } catch (_) { /* closed/cross-origin shadow root — skip */ }
+        });
+      } catch (_) { /* element threw — skip this root */ }
+    }
+    return Array.from(seen);
+  }, origin).catch(() => []);
+}
+
+/** Extract label→href pairs for all in-app anchor links — used to seed KAG element transitions. */
+async function extractLinkMappings(page: any, origin: string): Promise<Array<{ label: string; href: string }>> {
+  return page.evaluate((origin: string) => {
+    const logout = /logout|log out|sign out|delete/i;
+    return Array.from(document.querySelectorAll('a[href]'))
+      .filter((a: any) => !logout.test(a.textContent ?? ''))
+      .map((a: any) => {
+        const label = (
+          a.getAttribute('aria-label') ||
+          a.textContent?.replace(/\s+/g, ' ').trim() ||
+          a.getAttribute('title') ||
+          ''
+        ).slice(0, 80);
+        const href = (() => { try { const u = new URL(a.href); if (!/^#!?\//.test(u.hash)) u.hash = ''; return u.toString(); } catch { return ''; } })();
+        return { label, href };
+      })
+      .filter(({ label, href }: { label: string; href: string }) =>
+        label.length > 2 && href.length > 0 &&
+        (() => { try { return new URL(href).origin === origin; } catch { return false; } })()
+      );
+  }, origin).catch(() => []);
+}
+
+/** 
+ * Computes a structural hash of the DOM to detect repeating screens/components.
+ * Strips out text and focuses only on the sequence of semantic/form tags.
+ */
+async function computeDomHash(page: any): Promise<string> {
+  return page.evaluate(() => {
+    let hashStr = '';
+    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
+    let node;
+    while ((node = walker.nextNode())) {
+      const el = node as HTMLElement;
+      // Only include structural/semantic tags that matter for UI layouts
+      if (/^(MAIN|SECTION|NAV|FORM|INPUT|SELECT|TEXTAREA|BUTTON|TABLE|TR|TD|TH|UL|LI|A|H1|H2|H3)$/.test(el.tagName)) {
+        // Count structural nesting depth — distinguishes modal dialogs (deeply nested)
+        // from top-level nav/forms (shallow), preventing false duplicate matches.
+        let depth = 0;
+        let parent = el.parentElement;
+        while (parent) {
+          if (/^(MAIN|SECTION|NAV|FORM|INPUT|SELECT|TEXTAREA|BUTTON|TABLE|TR|TD|TH|UL|LI|A|H1|H2|H3)$/.test(parent.tagName)) depth++;
+          parent = parent.parentElement;
+        }
+        hashStr += el.tagName + depth + '|';
+      }
+    }
+    // Very simple hashing function (djb2 style)
+    let hash = 5381;
+    for (let i = 0; i < hashStr.length; i++) {
+      hash = (hash * 33) ^ hashStr.charCodeAt(i);
+    }
+    return (hash >>> 0).toString(16);
+  }).catch(() => '');
+}
+
+async function waitForURLChange(page: any, fromUrl: string, timeout = 10_000): Promise<void> {
+  const deadline = Date.now() + timeout;
+  while (Date.now() < deadline) {
+    const current = page.url();
+    try {
+      const from = new URL(fromUrl);
+      const now = new URL(current);
+      if (now.pathname !== from.pathname || now.origin !== from.origin) return;
+    } catch {
+      if (current !== fromUrl) return;
+    }
+    await new Promise(r => setTimeout(r, 200));
+  }
+  // Timed out — proceed with current URL (may still be login page)
+}
+
+/**
+ * Dismiss common overlay patterns (cookie banners, GDPR consent, chat widgets, newsletter modals)
+ * before element extraction — prevents banner elements from polluting the element model.
+ */
+async function dismissOverlays(page: any): Promise<void> {
+  const selectors = [
+    // OneTrust / CookieBot / TrustArc (most common GDPR banners)
+    '#onetrust-accept-btn-handler',
+    '#accept-cookie-consent',
+    '.cc-btn.cc-allow',
+    '[data-cookiebanner] button',
+    '[id*="cookie-accept"]',
+    '[class*="cookie-accept"]',
+    '[id*="cookieAccept"]',
+    '[class*="cookieAccept"]',
+    'button[aria-label*="Accept cookies" i]',
+    'button[aria-label*="Accept all" i]',
+    'button[title*="Accept" i][class*="cookie" i]',
+    // Intercom / Drift / HubSpot chat
+    '[class*="intercom-close"]',
+    '#hubspot-messages-iframe-container [aria-label*="close" i]',
+    // Generic overlay close buttons
+    '[role="dialog"] button[aria-label="Close" i]',
+    '[role="dialog"] button[aria-label="Dismiss" i]',
+    // Newsletter popups
+    'button[class*="modal-close"]',
+    'button[class*="popup-close"]',
+    '[class*="newsletter"] button[class*="close"]',
+  ];
+  for (const sel of selectors) {
+    try {
+      const el = page.locator(sel).first();
+      if (await el.isVisible({ timeout: 300 }).catch(() => false)) {
+        await el.click({ timeout: 500, force: true }).catch(() => { });
+      }
+    } catch { /* overlay not present — skip */ }
+  }
+  // Brief settle after dismissals
+  await page.waitForTimeout(300).catch(() => { });
+}
+
+/**
+ * Scroll the full page height to trigger IntersectionObserver-based lazy loads
+ * (images, components, infinite-scroll content). Scrolls back to top before capture.
+ */
+async function scrollToTriggerLazyLoad(page: any): Promise<void> {
+  try {
+    await page.evaluate(async () => {
+      const step = Math.max(window.innerHeight, 400);
+      const total = Math.min(document.body.scrollHeight, 12_000); // cap at 12k px to avoid infinite pages
+      for (let y = step; y < total; y += step) {
+        window.scrollTo(0, y);
+        await new Promise<void>(r => setTimeout(r, 120));
+      }
+      window.scrollTo(0, 0);
+    });
+    // Let lazy-triggered content render before extraction
+    await page.waitForTimeout(400).catch(() => { });
+    // Re-check DOM stability after scroll-triggered mutations
+    await waitForStableDOM(page, 3_000, 300);
+  } catch { /* scroll failed — non-fatal, proceed */ }
+}
+
+async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400): Promise<void> {
+  const deadline = Date.now() + maxWait;
+  let lastCount = -1;
+  let stableAt = 0;
+  while (Date.now() < deadline) {
+    const count: number = await page.evaluate(() => document.querySelectorAll('*').length).catch(() => 0);
+    if (count !== lastCount) {
+      lastCount = count;
+      stableAt = Date.now();
+    } else if (Date.now() - stableAt >= stabilityMs) {
+      return;
+    }
+    await new Promise(r => setTimeout(r, 100));
+  }
+  // Timed out — DOM may still be loading, proceed anyway
+}
+
+async function waitForScreenshotReady(page: any): Promise<void> {
+  try {
+    await page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { });
+    await page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { });
+    await page.evaluate(() => {
+      if (document.readyState === 'complete') return true;
+      return new Promise<boolean>((resolve) => {
+        const done = () => resolve(true);
+        window.addEventListener('load', done, { once: true });
+        setTimeout(done, 4_000);
+      });
+    }).catch(() => { });
+    await page.evaluate(async () => {
+      const fonts = (document as any).fonts;
+      if (fonts?.ready) await fonts.ready;
+    }).catch(() => { });
+    await scrollToTriggerLazyLoad(page);
+    await waitForStableDOM(page, 10_000, 700);
+    await page.evaluate(async () => {
+      const pending = Array.from(document.images).filter((img) => !img.complete);
+      if (pending.length === 0) return;
+      await Promise.race([
+        Promise.all(pending.map((img) => new Promise<void>((resolve) => {
+          img.addEventListener('load', () => resolve(), { once: true });
+          img.addEventListener('error', () => resolve(), { once: true });
+        }))),
+        new Promise<void>((resolve) => setTimeout(resolve, 4_000)),
+      ]);
+    }).catch(() => { });
+    await waitForStableDOM(page, 4_000, 500);
+    await page.waitForTimeout(250).catch(() => { });
+  } catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
+}
+
+async function hasMeaningfulAuthOrPublicState(page: any, requestedUrl: string): Promise<boolean> {
+  const pathLooksRelevant = (() => {
+    try {
+      const pathname = new URL(page.url?.() ?? requestedUrl).pathname;
+      return AUTH_PATH_RE.test(pathname) || PUBLIC_PATH_RE.test(pathname);
+    } catch {
+      return false;
+    }
+  })();
+
+  const domLooksRelevant = await page.evaluate(() => {
+    const text = (document.body?.innerText ?? '').toLowerCase();
+    const inputs = Array.from(document.querySelectorAll('input')) as HTMLInputElement[];
+    const hasPassword = inputs.some((i) => i.type === 'password');
+    const hasIdentityInput = inputs.some((i) => {
+      const meta = `${i.type} ${i.name} ${i.id} ${i.placeholder} ${i.autocomplete}`.toLowerCase();
+      return /(email|username|user|login|phone|otp|code|password)/.test(meta);
+    });
+    const hasAuthCopy = /(sign in|log in|login|sign up|create account|register|forgot password|reset password|verify email|one-time|otp|continue with email)/i.test(text);
+    const hasPublicNav = /(pricing|contact|features|documentation|docs|about|privacy|terms)/i.test(text);
+    return hasPassword || hasIdentityInput || hasAuthCopy || hasPublicNav;
+  }).catch(() => false);
+
+  return pathLooksRelevant || domLooksRelevant;
+}
+
+async function discoverAuthCandidates(page: any, origin: string): Promise<{
+  links: Array<{ href: string; label: string; flowType: AuthFlowType }>;
+  clicks: Array<{ selector: string; label: string; flowType: AuthFlowType }>;
+}> {
+  return page.evaluate((siteOrigin: string) => {
+    type BrowserFlowType = 'public' | 'login' | 'register' | 'password_reset' | 'verification' | 'invite' | 'session_recovery';
+    const authTextRe = /(sign\s*in|log\s*in|login|sign\s*up|signup|register|create account|forgot password|reset password|use email|continue with email|back to login|verify|invite|get started|start free trial)/i;
+    const skipTextRe = /(logout|log out|sign out|delete|remove|discard|cancel subscription|payment|billing)/i;
+    const authPathRe = /\/(login|log-in|signin|sign-in|auth|oauth|sso|account\/login|user\/login|app\/login|register|signup|sign-up|create-account|forgot|reset|password|verify-email|invite)(\/|$|\?)/i;
+    const publicPathRe = /\/($|pricing|contact|about|help|docs|features|security|privacy|terms)(\/|$|\?)/i;
+    const sameSiteBrowser = (href: string) => {
+      try {
+        const u = new URL(href, location.href);
+        const s = new URL(siteOrigin);
+        const strip = (h: string) => h.replace(/^www\./i, '');
+        return strip(u.hostname) === strip(s.hostname);
+      } catch {
+        return false;
+      }
+    };
+    const flowTypeFor = (value: string): BrowserFlowType => {
+      const v = value.toLowerCase();
+      if (/(forgot|reset|password)/.test(v)) return 'password_reset';
+      if (/(register|signup|sign up|create account|join)/.test(v)) return 'register';
+      if (/(verify|verification|otp|mfa|2fa|invite)/.test(v)) return v.includes('invite') ? 'invite' : 'verification';
+      if (/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)/.test(v)) return 'session_recovery';
+      if (/(login|log in|signin|sign in|auth|sso|oauth)/.test(v)) return 'login';
+      return 'public';
+    };
+    const textOf = (el: Element) => (
+      el.textContent ||
+      el.getAttribute('aria-label') ||
+      el.getAttribute('title') ||
+      el.getAttribute('placeholder') ||
+      ''
+    ).replace(/\s+/g, ' ').trim();
+    const isVisible = (el: Element) => {
+      const rect = (el as HTMLElement).getBoundingClientRect?.();
+      const style = window.getComputedStyle(el as HTMLElement);
+      return !!rect && rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
+    };
+    const links = new Map<string, { href: string; label: string; flowType: BrowserFlowType }>();
+    const addHref = (hrefRaw: string | null, labelRaw: string) => {
+      if (!hrefRaw || /^(javascript|mailto|tel):/i.test(hrefRaw)) return;
+      let href = '';
+      try { href = new URL(hrefRaw, location.href).toString(); } catch { return; }
+      if (!sameSiteBrowser(href)) return;
+      const u = new URL(href);
+      const label = labelRaw || u.pathname;
+      if (skipTextRe.test(label) || skipTextRe.test(u.pathname)) return;
+      if (!authTextRe.test(label) && !authPathRe.test(u.pathname) && !publicPathRe.test(u.pathname)) return;
+      const key = `${u.origin}${u.pathname.replace(/\/$/, '') || '/'}${u.search}`;
+      if (!links.has(key)) links.set(key, { href, label, flowType: flowTypeFor(`${label} ${u.pathname}`) });
+    };
+
+    const walk = (root: Document | ShadowRoot) => {
+      root.querySelectorAll('a[href]').forEach((el) => addHref((el as HTMLAnchorElement).href || el.getAttribute('href'), textOf(el)));
+      root.querySelectorAll('[data-href],[data-url],[data-link]').forEach((el: any) => addHref(el.dataset?.href || el.dataset?.url || el.dataset?.link || '', textOf(el)));
+      root.querySelectorAll('*').forEach((el: any) => { if (el.shadowRoot) walk(el.shadowRoot); });
+    };
+    walk(document);
+
+    const clicks: Array<{ selector: string; label: string; flowType: BrowserFlowType }> = [];
+    const clickEls = Array.from(document.querySelectorAll('button,[role="button"],a:not([href]),[data-state],[aria-haspopup]')) as HTMLElement[];
+    let idx = 0;
+    for (const el of clickEls) {
+      if (clicks.length >= 12) break;
+      if (!isVisible(el)) continue;
+      const label = textOf(el);
+      if (!label || !authTextRe.test(label) || skipTextRe.test(label)) continue;
+      const type = (el.getAttribute('type') || '').toLowerCase();
+      const inCredentialForm = !!el.closest('form')?.querySelector('input[type="password"],input[type="email"],input[name*="user" i],input[name*="email" i]');
+      const submitLike = type === 'submit' || /^(login|log in|sign in|continue)$/i.test(label);
+      // Avoid submitting empty login forms. Links/buttons that change auth state, such as
+      // "Forgot password" or "Create account", are safe to click even inside the form.
+      if (inCredentialForm && submitLike && !/(forgot|reset|register|sign up|create account|back to login|use email)/i.test(label)) continue;
+      const attr = `zeta-auth-${idx++}`;
+      el.setAttribute('data-zeta-auth-discovery-id', attr);
+      clicks.push({ selector: `[data-zeta-auth-discovery-id="${attr}"]`, label, flowType: flowTypeFor(label) });
+    }
+
+    return { links: Array.from(links.values()).slice(0, 32), clicks };
+  }, origin).catch(() => ({ links: [], clicks: [] }));
+}
+
+async function captureAuthDiscoveryScreen(
+  page: any,
+  opts: AuthDiscoveryOptions,
+  requestedUrl: string,
+  source: AuthDiscoverySource,
+  ordinal: number,
+  trigger?: { label?: string; flowType?: AuthFlowType; sameUrlState?: boolean },
+): Promise<{ captured: boolean; finalUrl: string; flowType: AuthFlowType }> {
+  const finalUrl = page.url?.() ?? requestedUrl;
+  if (finalUrl.startsWith('chrome-error://')) return { captured: false, finalUrl, flowType: 'public' };
+  if (!(await hasMeaningfulAuthOrPublicState(page, requestedUrl))) {
+    return { captured: false, finalUrl, flowType: flowTypeForUrlOrText(`${requestedUrl} ${finalUrl} ${trigger?.label ?? ''}`) };
+  }
+
+  const flowType = trigger?.flowType ?? flowTypeForUrlOrText(`${requestedUrl} ${finalUrl} ${trigger?.label ?? ''}`);
+  const title = await page.title().catch(() => '');
+  const baseName = title?.trim() && !/^(untitled|about:blank)$/i.test(title.trim())
+    ? title.trim().slice(0, 72)
+    : titleFromUrl(finalUrl);
+  const prefix = flowType === 'public' ? 'Public' : `Auth ${flowType.replace('_', ' ')}`;
+  const name = `${prefix} · ${trigger?.label ? trigger.label.slice(0, 48) : baseName}`.replace(/\s+/g, ' ');
+  const screenUrl = trigger?.sameUrlState
+    ? `${finalUrl.split('#')[0]}#zeta-auth-flow=${compactSlug(trigger.label ?? `${flowType}-${ordinal}`)}`
+    : finalUrl;
+  const shotPath = path.join(opts.screenshotDir ?? '/tmp', `auth-discovery-${opts.jobId ?? 'run'}-${ordinal}.png`);
+
+  await waitForScreenshotReady(page);
+  await page.screenshot({ path: shotPath, fullPage: true }).catch(() => { });
+  const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === 'entry' ? 'ENTRY_POINT' : 'SEED_URL').catch(() => null);
+  if (!screen) return { captured: false, finalUrl, flowType };
+  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP').catch(() => { });
+  await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+    screenId: screen.id,
+    jobId: opts.jobId,
+    type: 'AUTH_DISCOVERY',
+    payload: {
+      requestedUrl,
+      finalUrl,
+      screenUrl,
+      authState: flowType === 'public' ? 'logged_out' : 'auth_flow',
+      flowType,
+      source,
+      triggerLabel: trigger?.label ?? null,
+    },
+  }).catch(() => { });
+  logger.info({ requestedUrl, finalUrl, screenUrl, flowType, source }, '[crawler] auth-discovery: captured screen');
+  return { captured: true, finalUrl, flowType };
+}
+
+async function runAuthDiscoveryPhase(
+  page: any,
+  opts: AuthDiscoveryOptions,
+  origin: string,
+  onProgress?: AuthDiscoveryProgress,
+): Promise<{ captured: number; discoveredUrls: string[] }> {
+  const queue: Array<{ url: string; source: AuthDiscoverySource }> = [];
+  const queued = new Set<string>();
+  const capturedKeys = new Set<string>();
+  const discoveredUrls = new Set<string>();
+  const enqueue = (url: string, source: AuthDiscoverySource) => {
+    if (!shouldSeedAuthDiscoveryUrl(url, origin)) return;
+    const key = normalizeAuthUrl(url);
+    if (queued.has(key) || queue.length >= AUTH_DISCOVERY_MAX_QUEUE) return;
+    queued.add(key);
+    queue.push({ url, source });
+  };
+
+  enqueue(opts.appUrl, 'entry');
+  for (const p of AUTH_DISCOVERY_PATHS) {
+    try { enqueue(new URL(p, origin).toString(), 'seed_path'); } catch { /* ignore bad seed */ }
+  }
+
+  let captured = 0;
+  let ordinal = 0;
+  await onProgress?.(10, 'Discovering public and auth flows');
+
+  while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
+    const current = queue.shift()!;
+    try {
+      const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
+      if (response && response.status && response.status() >= 400) continue;
+      await page.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => { });
+      await waitForStableDOM(page);
+      await acceptCookieConsent(page, { waitMs: 400 }).catch(() => { });
+
+      const finalUrl = page.url?.() ?? current.url;
+      const captureKey = normalizeAuthUrl(finalUrl);
+      if (!capturedKeys.has(captureKey)) {
+        const result = await captureAuthDiscoveryScreen(page, opts, current.url, current.source, ordinal++);
+        if (result.captured) {
+          captured++;
+          capturedKeys.add(captureKey);
+          discoveredUrls.add(result.finalUrl);
+        }
+      }
+
+      const candidates = await discoverAuthCandidates(page, origin);
+      for (const link of candidates.links) enqueue(link.href, 'page_link');
+
+      const baseUrlForClicks = finalUrl;
+      const clickLimit = Math.min(candidates.clicks.length, AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE);
+      for (let i = 0; i < clickLimit && captured < AUTH_DISCOVERY_MAX_SCREENS; i++) {
+        await page.goto(baseUrlForClicks, { waitUntil: 'domcontentloaded', timeout: 12_000 }).catch(() => { });
+        await page.waitForLoadState('networkidle', { timeout: 4_000 }).catch(() => { });
+        await waitForStableDOM(page);
+        const freshCandidates = await discoverAuthCandidates(page, origin);
+        const click = freshCandidates.clicks[i];
+        if (!click) continue;
+        const beforeUrl = page.url?.() ?? baseUrlForClicks;
+        await page.locator(click.selector).first().click({ timeout: 4_000 }).catch(() => { });
+        await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
+        await waitForStableDOM(page);
+        const afterUrl = page.url?.() ?? beforeUrl;
+        const sameUrlState = normalizeAuthUrl(beforeUrl) === normalizeAuthUrl(afterUrl);
+        const stateKey = sameUrlState
+          ? `${normalizeAuthUrl(afterUrl)}::${compactSlug(click.label)}`
+          : normalizeAuthUrl(afterUrl);
+        if (capturedKeys.has(stateKey)) continue;
+        const result = await captureAuthDiscoveryScreen(page, opts, beforeUrl, 'auth_click', ordinal++, {
+          label: click.label,
+          flowType: click.flowType,
+          sameUrlState,
+        });
+        if (result.captured) {
+          captured++;
+          capturedKeys.add(stateKey);
+          discoveredUrls.add(result.finalUrl);
+          const nested = await discoverAuthCandidates(page, origin);
+          for (const link of nested.links) enqueue(link.href, 'page_link');
+        }
+      }
+    } catch (err) {
+      logger.warn({ url: current.url, err: String((err as Error)?.message ?? err) }, '[crawler] auth-discovery: URL failed — skipping');
+    }
+  }
+
+  await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => { });
+  await waitForStableDOM(page).catch(() => { });
+  await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+    jobId: opts.jobId,
+    type: 'AUTH_DISCOVERY_SUMMARY',
+    payload: { captured, discoveredUrls: Array.from(discoveredUrls), maxScreens: AUTH_DISCOVERY_MAX_SCREENS },
+  }).catch(() => { });
+  logger.info({ captured, discovered: discoveredUrls.size }, '[crawler] auth-discovery: phase complete');
+  return { captured, discoveredUrls: Array.from(discoveredUrls) };
+}