@detiq/crawler 0.1.3

package/dist/crawler.js

@@ -0,0 +1,3672 @@
+import { Stagehand } from "@browserbasehq/stagehand";
+import { chromium } from "rebrowser-playwright-core";
+import { chromium as stealthChromium } from "playwright-extra";
+import StealthPlugin from "puppeteer-extra-plugin-stealth";
+import { execSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+stealthChromium.use(StealthPlugin());
+import { z } from "zod";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { AppBrain, recordLLMCall } from "@qip/app-brain";
+import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from "@qip/agents";
+import { config } from "@qip/core";
+import { pHashVerdict, mergeAiVisualVerdict } from "./visual-diff-verdict.js";
+import { skyvernFallback, skyvernNavigate } from "./skyvern.js";
+import { loginWithStagehand } from "./ai-login.js";
+import { readScreen } from "./screen-reader.js";
+import { probeAllScreens } from "./interactive-probe.js";
+import { computeElementHash } from "./element-hash.js";
+import { logger } from "./logger.js";
+import { getSsoToken } from "./sso-token.js";
+import { discoverSitemapUrls, sameSite } from "./sitemap.js";
+import { extractMarkdown } from "./markdown-extract.js";
+import { CrawlFrontier } from "./frontier.js";
+import { RobotsCache } from "./robots-txt.js";
+import { classifyDomainProfile } from "./profiles/domain-profile.js";
+import { chooseCrawlEngines } from "./engine-router.js";
+import { assessActionSafety } from "./policies/crawl-safety.js";
+import { attachNetworkObserver } from "./network-observer.js";
+import { scrubPii } from "./pii-scrubber.js";
+import { attachWebSocketDetector } from "./websocket-detector.js";
+import { ContentDeduplicator } from "./content-dedup.js";
+import { guessGraphQLEndpoints, introspectGraphQL } from "./graphql-detector.js";
+import { acceptCookieConsent } from "./cookie-consent.js";
+import { detectJsRoutes } from "./js-router-detector.js";
+import { runGapAnalysis } from "./gap-pass.js";
+import { buildCrawlArtifactEnvelope, buildFitMarkdown, sha256, summarizeStructuredElements } from "./extractors/artifact-envelope.js";
+import { pickFreeProxy, TieredProxyManager } from "./proxy.js";
+import { infiniteScrollToBottom, extractIframeContent, pierceShadowDom, injectFingerprintOverrides } from "./stealth/human-behavior.js";
+import { checkSeo } from "./seo-checker.js";
+import { auditColorContrast } from "./color-contrast.js";
+import { detectFeatureFlags } from "./feature-flag-detector.js";
+import { detectLanguage } from "./language-detector.js";
+import { detectForms } from "./form-detector.js";
+import { RedirectRegistry } from "./redirect-tracker.js";
+import { DeadLinkTracker } from "./dead-link-detector.js";
+import { screensToSitemapUrls, writeSitemap } from "./sitemap-writer.js";
+import { generateOpenApiSpec, specToYaml } from "./openapi-generator.js";
+import { WebhookNotifier } from "./webhook-notifier.js";
+import { generatePlaywrightTest } from "./playwright-recorder.js";
+function getGitInfo() {
+  try {
+    const branch = execSync("git rev-parse --abbrev-ref HEAD", { encoding: "utf8", timeout: 3e3 }).trim();
+    const commit = execSync("git rev-parse HEAD", { encoding: "utf8", timeout: 3e3 }).trim().slice(0, 12);
+    return { branch: branch || "main", commit };
+  } catch {
+    return { branch: "main", commit: "" };
+  }
+}
+const ALLOWED_NUMERIC_PATHS = /* @__PURE__ */ new Set(["/2fa", "/404", "/500", "/403", "/401"]);
+const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
+const NUMERIC_ID_PATH_SEGMENT_RE = /^\d[\d-]*$/;
+const COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE = /^\d+-\d+$/;
+function normalizePathForNumericGuard(pathname) {
+  const normalized = pathname.replace(/\/+$/, "") || "/";
+  return normalized;
+}
+function isGeneratedPathSegment(segment) {
+  return UUID_PATH_SEGMENT_RE.test(segment) || CUID_PATH_SEGMENT_RE.test(segment) || COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE.test(segment) || NUMERIC_ID_PATH_SEGMENT_RE.test(segment);
+}
+function isGeneratedDynamicPath(pathname) {
+  const normalized = normalizePathForNumericGuard(pathname);
+  if (ALLOWED_NUMERIC_PATHS.has(normalized)) return false;
+  return normalized.split("/").filter(Boolean).some(isGeneratedPathSegment);
+}
+function isUsefulNavigationUrl(href, siteOrigin) {
+  try {
+    const u = new URL(href);
+    if (!sameSite(u.toString(), siteOrigin)) return false;
+    if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
+    if (isGeneratedDynamicPath(u.pathname)) return false;
+    return true;
+  } catch {
+    return false;
+  }
+}
+const AUTH_DISCOVERY_MAX_SCREENS = 36;
+const AUTH_DISCOVERY_MAX_QUEUE = 48;
+const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = 6;
+const AUTH_DISCOVERY_PATHS = [
+  "/",
+  "/login",
+  "/log-in",
+  "/signin",
+  "/sign-in",
+  "/auth",
+  "/auth/login",
+  "/account/login",
+  "/user/login",
+  "/app/login",
+  "/register",
+  "/signup",
+  "/sign-up",
+  "/create-account",
+  "/join",
+  "/forgot-password",
+  "/forgot",
+  "/password/forgot",
+  "/reset-password",
+  "/password/reset",
+  "/verify-email",
+  "/invite",
+  "/pricing",
+  "/contact",
+  "/about",
+  "/help",
+  "/docs"
+];
+const AUTH_PATH_RE = /\/(login|log-in|signin|sign-in|auth|oauth|sso|account\/login|user\/login|app\/login|register|signup|sign-up|create-account|forgot|reset|password|verify-email|invite)(\/|$|\?)/i;
+const PUBLIC_PATH_RE = /\/($|pricing|contact|about|help|docs|features|security|privacy|terms)(\/|$|\?)/i;
+function compactSlug(input, fallback = "state") {
+  const slug = input.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48);
+  return slug || fallback;
+}
+function titleFromUrl(url, fallback = "Page") {
+  try {
+    const u = new URL(url);
+    const parts = u.pathname.split("/").filter(Boolean);
+    if (parts.length === 0) return "Home";
+    return parts.slice(-2).join(" ").replace(/[-_]+/g, " ").replace(/\b\w/g, (m) => m.toUpperCase());
+  } catch {
+    return fallback;
+  }
+}
+function flowTypeForUrlOrText(value) {
+  const v = value.toLowerCase();
+  if (/(forgot|reset|password)/.test(v)) return "password_reset";
+  if (/(register|signup|sign up|create account|join)/.test(v)) return "register";
+  if (/(verify|verification|otp|mfa|2fa|invite)/.test(v)) return v.includes("invite") ? "invite" : "verification";
+  if (/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)/.test(v)) return "session_recovery";
+  if (/(login|log in|signin|sign in|auth|sso|oauth)/.test(v)) return "login";
+  return "public";
+}
+function shouldSeedAuthDiscoveryUrl(url, origin) {
+  try {
+    const u = new URL(url);
+    if (!sameSite(u.toString(), origin)) return false;
+    if (/\/(logout|logged-out|signout|sign-out|delete|remove|destroy)(\/|$|\?)/i.test(u.pathname)) return false;
+    return AUTH_PATH_RE.test(u.pathname) || PUBLIC_PATH_RE.test(u.pathname);
+  } catch {
+    return false;
+  }
+}
+function normalizeAuthUrl(url) {
+  try {
+    const u = new URL(url);
+    u.hash = "";
+    return `${u.origin}${u.pathname.replace(/\/$/, "") || "/"}${u.search}`;
+  } catch {
+    return url;
+  }
+}
+async function capturePageHtml(page) {
+  try {
+    if (typeof page.content === "function") {
+      const html = await page.content();
+      if (typeof html === "string" && html.trim()) return html;
+    }
+  } catch (err) {
+    logger.warn({ err: String(err?.message ?? err) }, "[crawler] page.content() failed; trying DOM evaluate fallback");
+  }
+  try {
+    const html = await page.evaluate(() => {
+      const type = document.doctype;
+      const doctype = type ? "<!DOCTYPE " + type.name + (type.publicId ? ' PUBLIC "' + type.publicId + '"' : "") + (type.systemId ? ' "' + type.systemId + '"' : "") + ">" : "<!DOCTYPE html>";
+      return doctype + "\n" + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? "");
+    });
+    return typeof html === "string" && html.trim() ? html : null;
+  } catch (err) {
+    logger.warn({ err: String(err?.message ?? err) }, "[crawler] DOM snapshot capture failed");
+    return null;
+  }
+}
+async function launchChromeCDP(execPath, extraArgs, proxy) {
+  const port = 9200 + Math.floor(Math.random() * 700);
+  const browserServer = await stealthChromium.launchServer({
+    headless: true,
+    executablePath: execPath,
+    // Use Playwright's native proxy option — it handles 407 auth challenges automatically.
+    // The --proxy-server Chrome flag does NOT support credentials with special chars (commas).
+    ...proxy ? { proxy } : {},
+    args: [
+      ...extraArgs,
+      `--remote-debugging-port=${port}`,
+      "--remote-allow-origins=*",
+      "--no-first-run",
+      "--no-default-browser-check"
+    ]
+  });
+  const deadline = Date.now() + 1e4;
+  while (Date.now() < deadline) {
+    await new Promise((r) => setTimeout(r, 300));
+    try {
+      const res = await fetch(`http://127.0.0.1:${port}/json/version`);
+      if (res.ok) {
+        const { webSocketDebuggerUrl } = await res.json();
+        return { wsUrl: webSocketDebuggerUrl, proc: browserServer };
+      }
+    } catch {
+    }
+  }
+  await browserServer.close();
+  throw new Error(`Chrome failed to open CDP port ${port} within 10 s`);
+}
+async function discoverLinksViaBrowser(appUrl, opts) {
+  let proxy;
+  const purl = opts?.proxyUrl;
+  if (purl && purl !== "auto") {
+    const m = purl.match(/^(https?|socks5?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (m) {
+      const [, scheme, user, pass, hostPort] = m;
+      proxy = user ? { server: `${scheme}://${hostPort}`, username: decodeURIComponent(user), password: decodeURIComponent(pass ?? "") } : { server: `${scheme}://${hostPort}` };
+    } else {
+      proxy = { server: purl };
+    }
+  }
+  let executablePath;
+  try {
+    executablePath = chromium.executablePath();
+  } catch {
+  }
+  let browser = null;
+  try {
+    browser = await chromium.launch({
+      headless: true,
+      ...executablePath ? { executablePath } : {},
+      ...proxy ? { proxy } : {},
+      args: ["--no-sandbox", "--disable-dev-shm-usage"]
+    });
+    const page = await browser.newPage();
+    await page.goto(appUrl, { waitUntil: "networkidle", timeout: 3e4 }).catch(() => {
+    });
+    await page.waitForTimeout(2500);
+    const hrefs = await page.evaluate((origin) => {
+      const found = /* @__PURE__ */ new Set();
+      const walk = (root) => {
+        root.querySelectorAll("a[href]").forEach((a) => {
+          const h = a.href || a.getAttribute("href") || "";
+          if (h && (h.startsWith(origin) || h.startsWith("/"))) found.add(h);
+        });
+        root.querySelectorAll("*").forEach((el) => {
+          if (el.shadowRoot) walk(el.shadowRoot);
+        });
+      };
+      walk(document);
+      return [...found];
+    }, new URL(appUrl).origin).catch(() => []);
+    return [...new Set(hrefs.filter((h) => sameSite(h, appUrl)))].slice(0, opts?.maxLinks ?? 500);
+  } catch {
+    return [];
+  } finally {
+    if (browser) await browser.close().catch(() => {
+    });
+  }
+}
+async function runLighthouse(url) {
+  if (!process.env.ENABLE_LIGHTHOUSE) return null;
+  try {
+    const { default: lighthouse } = await import("lighthouse");
+    const chromeLauncher = await import("chrome-launcher");
+    const chrome = await chromeLauncher.launch({
+      chromeFlags: ["--headless", "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"]
+    });
+    try {
+      const result = await lighthouse(url, {
+        port: chrome.port,
+        output: "json",
+        logLevel: "error",
+        onlyCategories: ["performance", "accessibility", "seo", "best-practices"]
+      });
+      if (!result?.lhr?.categories) return null;
+      const cats = result.lhr.categories;
+      const score = (key) => Math.round((cats[key]?.score ?? 0) * 100);
+      return {
+        perf: score("performance"),
+        a11y: score("accessibility"),
+        seo: score("seo"),
+        bp: score("best-practices"),
+        report: result.lhr
+      };
+    } finally {
+      await chrome.kill();
+    }
+  } catch (err) {
+    console.warn("[crawler] Lighthouse run failed (non-fatal):", err);
+    return null;
+  }
+}
+const ElementSchema = z.object({
+  elements: z.array(z.object({
+    meaning: z.string(),
+    role: z.string(),
+    expectedData: z.string().optional()
+  }))
+});
+const LinksSchema = z.object({
+  links: z.array(z.string())
+});
+const TwoFASchema = z.object({
+  has2FA: z.boolean(),
+  fieldLabel: z.string().optional()
+});
+async function crawlProject(opts, onProgress) {
+  if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
+  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === "production";
+  const toStagehandModel = (model, provider) => model.includes("/") ? model : `${provider}/${model}`;
+  let stagehandModel = toStagehandModel(config.llmModel, "openai");
+  let stagehandApiKey = "";
+  let configuredProvider = "";
+  try {
+    const crawlLLM = await getTaskAgentLLMConfig(opts.tenantId, "crawl");
+    if (crawlLLM) {
+      configuredProvider = crawlLLM.provider;
+      if (crawlLLM.apiKey && crawlLLM.model) {
+        if (crawlLLM.provider === "anthropic" || crawlLLM.provider === "openai") {
+          stagehandModel = toStagehandModel(crawlLLM.model, crawlLLM.provider);
+          stagehandApiKey = crawlLLM.apiKey;
+        } else if (crawlLLM.provider === "openrouter") {
+          const model = crawlLLM.model.includes("/") ? crawlLLM.model : `openai/${crawlLLM.model}`;
+          stagehandModel = model;
+          stagehandApiKey = crawlLLM.apiKey;
+        }
+      }
+    }
+  } catch {
+  }
+  if (!stagehandApiKey) {
+    const envModel = process.env.STAGEHAND_MODEL ?? "";
+    const preferAnthropic = envModel.includes("claude") || !envModel && !process.env.OPENAI_API_KEY;
+    if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
+      stagehandApiKey = process.env.ANTHROPIC_API_KEY;
+      stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || "claude-sonnet-4-6"}`;
+    } else if (process.env.OPENAI_API_KEY) {
+      stagehandApiKey = process.env.OPENAI_API_KEY;
+      stagehandModel = envModel.includes("/") ? envModel : `openai/${envModel || "gpt-4o"}`;
+    } else if (process.env.ANTHROPIC_API_KEY) {
+      stagehandApiKey = process.env.ANTHROPIC_API_KEY;
+      stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || "claude-sonnet-4-6"}`;
+    }
+  }
+  const elementLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, "elementExtraction").catch(() => null);
+  const linkLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, "linkDiscovery").catch(() => null);
+  const projectForProfile = await AppBrain.getProjectMap(opts.tenantId, opts.projectId).catch(() => null);
+  const profileProjectName = projectForProfile?.name ?? null;
+  const profileAppType = projectForProfile?.appType ?? null;
+  const domainProfile = classifyDomainProfile({
+    appUrl: opts.appUrl,
+    projectName: profileProjectName,
+    appType: profileAppType,
+    seedText: [
+      ...opts.startUrls ?? [],
+      opts.loginInstructions ?? "",
+      ...opts.bestFirstKeywords ?? []
+    ].join(" ")
+  });
+  const routeDecision = chooseCrawlEngines({
+    appUrl: opts.appUrl,
+    appType: profileAppType,
+    hasCredentials: !!opts.credentials,
+    hasStorageState: !!opts.storageState,
+    hasAuthToken: !!opts.authToken,
+    hasSkyvern: !!opts.skyvernConfig,
+    hasProxy: !!opts.proxyUrl,
+    profile: domainProfile
+  });
+  AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+    jobId: opts.jobId,
+    type: "CRAWL_PROFILE",
+    payload: { profile: domainProfile, routeDecision }
+  }).catch((err) => console.warn("[crawler] profile evidence save failed:", err?.message ?? err));
+  if (!stagehandApiKey) {
+    const unsupported = configuredProvider && !["anthropic", "openai", "openrouter"].includes(configuredProvider);
+    throw new Error(
+      unsupported ? `Browser crawling requires Anthropic or OpenAI. Your primary provider "${configuredProvider}" is not supported for crawling. Add a separate Crawl API Key in Workspace \u2192 Settings \u2192 AI Provider.` : "No AI provider configured. Go to Workspace \u2192 Settings \u2192 AI Provider to set up Anthropic or OpenAI for browser crawling."
+    );
+  }
+  const stealthArgs = [
+    "--disable-blink-features=AutomationControlled",
+    "--disable-http2",
+    "--lang=en-US,en",
+    "--window-size=1366,768",
+    "--disable-features=IsolateOrigins,site-per-process"
+  ];
+  const chromeArgs = isDocker ? [
+    "--no-sandbox",
+    "--disable-setuid-sandbox",
+    "--disable-dev-shm-usage",
+    "--disable-gpu",
+    ...stealthArgs
+  ] : [...stealthArgs];
+  let resolvedProxyUrl = opts.proxyUrl;
+  let tieredProxyManager;
+  if (opts.proxyTiers?.length) {
+    tieredProxyManager = new TieredProxyManager(opts.proxyTiers);
+    const domain = (() => {
+      try {
+        return new URL(opts.appUrl).hostname;
+      } catch {
+        return opts.appUrl;
+      }
+    })();
+    resolvedProxyUrl = tieredProxyManager.selectProxy(domain) ?? resolvedProxyUrl;
+    logger.info({ domain, proxyUrl: resolvedProxyUrl }, "[crawler] tiered proxy selected");
+  } else if (resolvedProxyUrl === "auto") {
+    logger.info("[crawler] proxyUrl=auto: selecting free proxy from public list...");
+    resolvedProxyUrl = await pickFreeProxy() ?? void 0;
+    if (resolvedProxyUrl) {
+      logger.info({ proxyUrl: resolvedProxyUrl }, "[crawler] auto-proxy selected");
+    } else {
+      logger.warn("[crawler] auto-proxy: no working proxy found, proceeding without proxy");
+    }
+  }
+  let playwrightProxy;
+  if (resolvedProxyUrl) {
+    try {
+      const proxyMatch = resolvedProxyUrl.match(/^(https?|socks5?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+      if (proxyMatch) {
+        const [, scheme, rawUser, rawPass, hostPort] = proxyMatch;
+        const server = `${scheme}://${hostPort}`;
+        const username = rawUser ? decodeURIComponent(rawUser) : void 0;
+        const password = rawPass ? decodeURIComponent(rawPass) : void 0;
+        playwrightProxy = username ? { server, username, password } : { server };
+        logger.info({ server, hasAuth: !!username }, "[crawler] proxy configured (Playwright native)");
+      } else {
+        playwrightProxy = { server: resolvedProxyUrl };
+      }
+    } catch {
+      playwrightProxy = { server: resolvedProxyUrl };
+    }
+  }
+  let executablePath;
+  try {
+    executablePath = chromium.executablePath();
+  } catch {
+  }
+  if (!executablePath) throw new Error("No Chrome binary found. Set CHROME_PATH or install Playwright browsers.");
+  let effectiveStorageState = opts.storageState;
+  if (opts.chromeProfilePath && !isDocker) {
+    logger.info({ chromeProfilePath: opts.chromeProfilePath }, "[crawler] Exporting session from Chrome profile");
+    try {
+      const tempCtx = await chromium.launchPersistentContext(opts.chromeProfilePath, {
+        headless: true,
+        args: [...chromeArgs, "--no-first-run", "--no-default-browser-check"],
+        executablePath
+      });
+      const exportedState = await tempCtx.storageState();
+      await tempCtx.close().catch(() => {
+      });
+      if ((exportedState.cookies?.length ?? 0) > 0 || (exportedState.origins?.length ?? 0) > 0) {
+        effectiveStorageState = JSON.stringify(exportedState);
+        logger.info({ cookies: exportedState.cookies?.length ?? 0, origins: exportedState.origins?.length ?? 0 }, "[crawler] Chrome profile session exported and injected");
+      } else {
+        logger.warn("[crawler] Chrome profile had no session data \u2014 continuing without profile auth");
+      }
+    } catch (profileErr) {
+      logger.warn({ err: profileErr }, "[crawler] Could not load Chrome profile \u2014 continuing without profile auth");
+    }
+  } else if (opts.chromeProfilePath && isDocker) {
+    logger.warn("[crawler] chromeProfilePath is set but will be ignored in Docker/cloud environments");
+  }
+  if (opts.useFirefox) {
+    logger.info("[crawler] useFirefox: launching Camoufox to bootstrap Cloudflare cookies...");
+    try {
+      const { NewBrowser: camoufoxNewBrowser } = await import("camoufox");
+      const pw = require("playwright");
+      const foxBrowser = await camoufoxNewBrowser(pw, true);
+      const foxPage = await foxBrowser.newPage();
+      await foxPage.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
+      await foxPage.waitForTimeout(8e3);
+      const foxState = await foxPage.context().storageState();
+      await foxBrowser.close().catch(() => {
+      });
+      if ((foxState.cookies?.length ?? 0) > 0) {
+        if (effectiveStorageState) {
+          try {
+            const existing = JSON.parse(effectiveStorageState);
+            const merged = { ...existing, cookies: [...existing.cookies ?? [], ...foxState.cookies] };
+            effectiveStorageState = JSON.stringify(merged);
+          } catch {
+            effectiveStorageState = JSON.stringify(foxState);
+          }
+        } else {
+          effectiveStorageState = JSON.stringify(foxState);
+        }
+        logger.info({ cookies: foxState.cookies.length }, "[crawler] Camoufox bootstrap complete \u2014 cf_clearance injected into Chromium");
+      } else {
+        logger.warn("[crawler] Camoufox returned no cookies \u2014 continuing without bootstrap");
+      }
+    } catch (foxErr) {
+      logger.warn({ err: foxErr }, "[crawler] Camoufox bootstrap failed \u2014 continuing without it");
+    }
+  }
+  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy);
+  let stagehandPromptTokens = 0;
+  let stagehandCompletionTokens = 0;
+  const stagehandLogger = (line) => {
+    if (line.message === "response" && line.auxiliary?.response?.value) {
+      try {
+        const resp = JSON.parse(line.auxiliary.response.value);
+        if (resp?.usage) {
+          stagehandPromptTokens += resp.usage.prompt_tokens ?? 0;
+          stagehandCompletionTokens += resp.usage.completion_tokens ?? 0;
+        }
+      } catch {
+      }
+    }
+  };
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    model: { modelName: stagehandModel, apiKey: stagehandApiKey },
+    serverCache: true,
+    verbose: 1,
+    logger: stagehandLogger,
+    localBrowserLaunchOptions: { cdpUrl, headless: true },
+    domSettleTimeout: 2e3
+  });
+  await stagehand.init();
+  await onProgress?.(5, "Browser launched");
+  const page = await stagehand.resolvePage();
+  try {
+    page.setDefaultTimeout(45e3);
+  } catch {
+  }
+  try {
+    page.setDefaultNavigationTimeout(3e4);
+  } catch {
+  }
+  let rawPage = null;
+  await page.addInitScript(() => {
+    Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+    Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+    const fakePlugins = ["PDF Viewer", "Chrome PDF Viewer", "Chromium PDF Viewer", "Microsoft Edge PDF Viewer", "WebKit built-in PDF"];
+    Object.defineProperty(navigator, "plugins", {
+      get: () => Object.assign(fakePlugins.map((name, i) => ({ name, filename: "internal-pdf-viewer", description: "", length: 1, item: () => null, namedItem: () => null })), { item: (i) => null, namedItem: () => null, refresh: () => {
+      } })
+    });
+    window.chrome = { runtime: {}, loadTimes: () => ({}), csi: () => ({}), app: {} };
+    const origQuery = window.navigator.permissions?.query?.bind(navigator.permissions);
+    if (origQuery) {
+      navigator.permissions.query = (params) => params?.name === "notifications" ? Promise.resolve({ state: "prompt", onchange: null }) : origQuery(params);
+    }
+    try {
+      Object.defineProperty(Notification, "permission", { get: () => "default" });
+    } catch {
+    }
+    if (!navigator.connection) Object.defineProperty(navigator, "connection", { get: () => ({ rtt: 50, downlink: 10, effectiveType: "4g", saveData: false }) });
+    delete window.__playwright;
+    delete window.__pw_manual;
+    delete window.__PW_inspect;
+  });
+  await page.addInitScript(() => {
+    try {
+      window.__zetaLCP = null;
+      window.__zetaCLS = 0;
+      new PerformanceObserver((list) => {
+        const entries = list.getEntries();
+        const last = entries[entries.length - 1];
+        if (last) window.__zetaLCP = last.startTime;
+      }).observe({ type: "largest-contentful-paint", buffered: true });
+      new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          if (!entry.hadRecentInput) window.__zetaCLS += entry.value;
+        }
+      }).observe({ type: "layout-shift", buffered: true });
+    } catch {
+    }
+  });
+  if (opts.httpBasicUsername && opts.httpBasicPassword) {
+    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString("base64");
+    const appOrigin = new URL(opts.appUrl).origin;
+    await page.route("**/*", (route) => {
+      const reqUrl = route.request().url();
+      if (reqUrl.startsWith(appOrigin)) {
+        route.continue({
+          headers: { ...route.request().headers(), "Authorization": `Basic ${encoded}` }
+        });
+      } else {
+        route.continue();
+      }
+    });
+  }
+  if (opts.authToken) {
+    const headerName = opts.authTokenHeader ?? "Authorization";
+    const prefix = opts.authTokenPrefix ?? "Bearer ";
+    await page.setExtraHTTPHeaders({ [headerName]: `${prefix}${opts.authToken}` });
+    await page.addInitScript((token) => {
+      localStorage.setItem("auth_token", token);
+      localStorage.setItem("access_token", token);
+      localStorage.setItem("token", token);
+    }, opts.authToken);
+  }
+  await page.addInitScript(() => {
+    try {
+      window.__zetaCLS = 0;
+      new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          if (!entry.hadRecentInput) window.__zetaCLS += entry.value;
+        }
+      }).observe({ type: "layout-shift", buffered: true });
+    } catch {
+    }
+  });
+  await page.addInitScript(() => {
+    window.__zetaDiscoveredRoutes = window.__zetaDiscoveredRoutes ?? [];
+    const push = history.pushState.bind(history);
+    const replace = history.replaceState.bind(history);
+    history.pushState = function(data, unused, url) {
+      const result = push(data, unused, url);
+      if (url && typeof url === "string") window.__zetaDiscoveredRoutes.push(url);
+      return result;
+    };
+    history.replaceState = function(data, unused, url) {
+      const result = replace(data, unused, url);
+      if (url && typeof url === "string") window.__zetaDiscoveredRoutes.push(url);
+      return result;
+    };
+  });
+  await injectFingerprintOverrides(page).catch(() => {
+  });
+  const capturedApiEndpoints = [];
+  const revisitQueue = [];
+  const xhrDiscoveredPaths = /* @__PURE__ */ new Set();
+  let playwrightBrowser = null;
+  try {
+    playwrightBrowser = await chromium.connectOverCDP(cdpUrl);
+    const pwContext = playwrightBrowser.contexts()[0];
+    if (pwContext) {
+      attachNetworkObserver(pwContext, capturedApiEndpoints, (relPath) => xhrDiscoveredPaths.add(relPath));
+      pwContext.on("page", (popup) => {
+        const popupUrl = popup.url();
+        logger.info({ url: popupUrl }, "[crawler] OAuth popup detected \u2014 waiting for close");
+        popup.waitForEvent("close", { timeout: 12e4 }).then(() => logger.info("[crawler] OAuth popup closed \u2014 SSO flow complete")).catch(() => logger.warn("[crawler] OAuth popup timed out or errored"));
+      });
+    }
+    const pwContext2 = playwrightBrowser?.contexts()?.[0];
+    if (pwContext2) {
+      const pwPages = pwContext2.pages();
+      rawPage = pwPages.find((p) => p.url() !== "about:blank") ?? pwPages[0] ?? rawPage;
+    }
+    if (playwrightProxy?.username && pwContext2 && rawPage && rawPage !== page) {
+      try {
+        const proxyCdp = await pwContext2.newCDPSession(rawPage);
+        await proxyCdp.send("Fetch.enable", { handleAuthRequests: true, patterns: [{ urlPattern: "*" }] });
+        proxyCdp.on("Fetch.requestPaused", (e) => {
+          proxyCdp.send("Fetch.continueRequest", { requestId: e.requestId }).catch(() => {
+          });
+        });
+        let proxyAuthed = false;
+        proxyCdp.on("Fetch.authRequired", async (e) => {
+          const isProxy = e.authChallenge?.source === "Proxy";
+          try {
+            await proxyCdp.send("Fetch.continueWithAuth", {
+              requestId: e.requestId,
+              authChallengeResponse: isProxy ? { response: "ProvideCredentials", username: playwrightProxy.username, password: playwrightProxy.password ?? "" } : { response: "Default" }
+            });
+          } catch {
+          }
+          if (isProxy && !proxyAuthed) {
+            proxyAuthed = true;
+            proxyCdp.send("Fetch.disable").catch(() => {
+            });
+          }
+        });
+        logger.info({ server: playwrightProxy.server }, "[crawler] proxy auth via CDP Fetch handler enabled");
+      } catch (e) {
+        logger.warn({ err: e?.message }, "[crawler] CDP proxy-auth setup failed \u2014 407 may not resolve");
+      }
+    }
+  } catch {
+  }
+  if (!rawPage) rawPage = page;
+  try {
+    await onProgress?.(8, "Loading app");
+    await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(async () => {
+      await page.goto(opts.appUrl, { waitUntil: "commit", timeout: 15e3 }).catch(() => {
+      });
+    });
+    await acceptCookieConsent(page, { waitMs: 1e3 }).catch(() => {
+    });
+    if (page.url().startsWith("chrome-error://")) {
+      throw new Error(`Cannot reach ${opts.appUrl} \u2014 the server refused the connection (likely IP block or DNS failure). Go to Project Settings \u2192 Connectivity \u2192 Proxy and select "Auto" or enter a residential proxy URL.`);
+    }
+    const initialTitle = await page.title().catch(() => "");
+    const isBotChallenge = /^(just a moment|attention required|access denied|ddos protection|cloudflare|checking your browser|enable javascript)/i.test(initialTitle);
+    if (isBotChallenge) {
+      throw new Error(`Cannot crawl ${opts.appUrl} \u2014 bot protection challenge detected (title: "${initialTitle}"). The site is blocking automated browsers. Go to Project Settings \u2192 Connectivity \u2192 Proxy and select "Auto" (free proxies) or enter a residential proxy URL to bypass.`);
+    }
+    await page.waitForLoadState("networkidle", { timeout: 1e4 }).catch(() => {
+    });
+    await waitForStableDOM(page);
+    if (!opts.discoveryMode) {
+      try {
+        await runAuthDiscoveryPhase(page, opts, new URL(opts.appUrl).origin, onProgress);
+      } catch (preAuthErr) {
+        logger.warn({ err: preAuthErr }, "[crawler] auth-discovery phase failed \u2014 continuing with authenticated crawl");
+      }
+    }
+    let didLogin = false;
+    if (opts.authToken) {
+      await onProgress?.(18, "API token configured \u2014 skipping login");
+      logger.info("authToken provided \u2014 skipping AI login");
+    } else if (effectiveStorageState) {
+      logger.info("StorageState found \u2014 restoring session, skipping AI login");
+      try {
+        const state = JSON.parse(effectiveStorageState);
+        if (state.cookies?.length) {
+          const browserCtx = playwrightBrowser?.contexts()[0];
+          if (browserCtx) {
+            await browserCtx.addCookies(state.cookies);
+          }
+        }
+        if (state.origins?.length) {
+          await rawPage.addInitScript((origins) => {
+            for (const entry of origins ?? []) {
+              if (window.location.origin === entry.origin) {
+                for (const item of entry.localStorage) {
+                  try {
+                    localStorage.setItem(item.name, item.value);
+                  } catch {
+                  }
+                }
+              }
+            }
+          }, state.origins);
+        }
+        await onProgress?.(12, "Restoring session");
+        await rawPage.reload({ waitUntil: "domcontentloaded" });
+        await waitForStableDOM(page);
+        if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, "")) {
+          await waitForURLChange(page, opts.appUrl).catch(() => {
+          });
+          await waitForStableDOM(page);
+        }
+        const postRestoreUrl = page.url();
+        const LOGIN_PATH = /\/(login|signin|sign-in|auth|log-in|sso|oauth|account\/login)(\/|$|\?)/i;
+        const ERROR_PATH = /\/(session-expired|expired|unauthorized|forbidden|401|403|error|logout|logged-out)(\/|$|\?)/i;
+        const redirectedToLogin = LOGIN_PATH.test(new URL(postRestoreUrl).pathname);
+        const redirectedToError = ERROR_PATH.test(new URL(postRestoreUrl).pathname);
+        const hasLoginForm = await page.evaluate(() => {
+          const inputs = Array.from(document.querySelectorAll("input"));
+          const hasEmailOrUser = inputs.some(
+            (i) => ["email", "username", "user", "login"].some(
+              (k) => (i.type || "").includes(k) || (i.name || "").toLowerCase().includes(k) || (i.id || "").toLowerCase().includes(k) || (i.placeholder || "").toLowerCase().includes(k)
+            )
+          );
+          const hasPasswordField = inputs.some((i) => i.type === "password");
+          return hasEmailOrUser && hasPasswordField;
+        }).catch(() => false);
+        const sessionFailed = redirectedToLogin || redirectedToError || hasLoginForm;
+        if (sessionFailed) {
+          logger.warn({ url: postRestoreUrl, loginForm: hasLoginForm, errorUrl: redirectedToError }, "StorageState restore failed \u2014 session may be expired");
+          await AppBrain.saveCrawlConfig(opts.tenantId, opts.projectId, { storageState: null }).catch(() => {
+          });
+          await page.goto(opts.appUrl, { waitUntil: "domcontentloaded" }).catch(() => {
+          });
+          await waitForStableDOM(page);
+          const urlAfterNav = page.url();
+          const stillAtError = ERROR_PATH.test(new URL(urlAfterNav).pathname) || LOGIN_PATH.test(new URL(urlAfterNav).pathname);
+          if (stillAtError) {
+            const appOrigin = new URL(opts.appUrl).origin;
+            for (const loginPath of ["/login", "/signin", "/sign-in", "/auth/login", "/auth"]) {
+              try {
+                await page.goto(`${appOrigin}${loginPath}`, { waitUntil: "domcontentloaded" });
+                const hasForm = await page.evaluate(() => {
+                  const inputs = Array.from(document.querySelectorAll("input"));
+                  return inputs.some((i) => i.type === "password");
+                }).catch(() => false);
+                if (hasForm) {
+                  logger.info({ loginPath }, "Found login form at path");
+                  break;
+                }
+              } catch {
+              }
+            }
+            await waitForStableDOM(page);
+          }
+          if (opts.credentials) {
+            logger.info("Falling back to AI login with credentials");
+            await onProgress?.(15, "Session expired \u2014 logging in with credentials\u2026");
+            const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+            if (!loggedIn) throw new Error("[SESSION_EXPIRED] Stored session expired and credential login also failed. Check your username / password in Project Settings \u2192 Credentials.");
+            didLogin = true;
+          } else {
+            throw new Error("[SESSION_EXPIRED] Stored session has expired. Please re-capture your session in Project Settings \u2192 Advanced: Session Capture, or add username / password as a login fallback.");
+          }
+        } else {
+          await onProgress?.(18, "Session restored");
+          logger.info({ url: postRestoreUrl }, "Session restored");
+        }
+      } catch (err) {
+        const msg = String(err.message ?? err);
+        if (msg.includes("[SESSION_EXPIRED]") || msg.includes("expired")) throw err;
+        logger.warn({ err }, "StorageState restore failed, falling back to AI login");
+        if (opts.credentials) {
+          await onProgress?.(15, "Session restore failed \u2014 logging in with credentials\u2026");
+          const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+          if (!loggedIn) throw new Error("[SESSION_EXPIRED] Session restore failed and credential login also failed. Check your username / password in Project Settings \u2192 Credentials.");
+          didLogin = true;
+        } else {
+          throw new Error("[SESSION_EXPIRED] Session restore failed and no credentials were provided. Please re-capture your session in Project Settings \u2192 Advanced: Session Capture.");
+        }
+      }
+    } else if (opts.ssoProvider && opts.ssoDomain && opts.ssoClientId && opts.credentials) {
+      await onProgress?.(12, "SSO: fetching token from identity provider");
+      try {
+        const ssoToken = await getSsoToken(
+          {
+            provider: opts.ssoProvider,
+            domain: opts.ssoDomain,
+            clientId: opts.ssoClientId,
+            clientSecret: opts.ssoClientSecret,
+            scope: opts.ssoScope
+          },
+          opts.credentials.username,
+          opts.credentials.password
+        );
+        await page.setExtraHTTPHeaders({ "Authorization": `Bearer ${ssoToken}` });
+        await page.addInitScript((t) => {
+          localStorage.setItem("access_token", t);
+          localStorage.setItem("auth_token", t);
+          localStorage.setItem("token", t);
+        }, ssoToken);
+        await page.reload({ waitUntil: "domcontentloaded" });
+        didLogin = true;
+        await onProgress?.(18, "SSO token injected");
+        logger.info({ provider: opts.ssoProvider }, "[crawler] SSO token obtained and injected");
+      } catch (ssoErr) {
+        logger.warn({ err: ssoErr }, "[crawler] SSO token retrieval failed \u2014 falling back to browser login");
+        await onProgress?.(12, "Logging in");
+        didLogin = true;
+        const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+        if (!loggedIn) {
+          logger.warn("Stagehand login failed \u2014 trying Skyvern vision fallback");
+          await onProgress?.(14, "Trying vision fallback");
+          const skyvernResult = await skyvernFallback({
+            taskDescription: `Log in to ${opts.appUrl} using username "${opts.credentials.username}" and navigate to the main dashboard.`,
+            url: opts.appUrl,
+            credentials: opts.credentials
+          }, opts.skyvernConfig);
+          if (skyvernResult === "failed") {
+            throw new Error(
+              "Login failed on both Stagehand (DOM) and Skyvern (vision). Check credentials. For CAPTCHA/SSO sites, set SKYVERN_API_URL + SKYVERN_API_KEY."
+            );
+          }
+        }
+        await onProgress?.(18, "Logged in");
+      }
+    } else if (opts.credentials) {
+      await onProgress?.(12, "Logging in");
+      didLogin = true;
+      const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
+      if (!loggedIn) {
+        logger.warn("Stagehand login failed \u2014 trying Skyvern vision fallback");
+        await onProgress?.(14, "Trying vision fallback");
+        const skyvernResult = await skyvernFallback({
+          taskDescription: `Log in to ${opts.appUrl} using username "${opts.credentials.username}" and navigate to the main dashboard.`,
+          url: opts.appUrl,
+          credentials: opts.credentials
+        }, opts.skyvernConfig);
+        if (skyvernResult === "failed") {
+          throw new Error(
+            "Login failed on both Stagehand (DOM) and Skyvern (vision). Check credentials. For CAPTCHA/SSO sites, set SKYVERN_API_URL + SKYVERN_API_KEY."
+          );
+        }
+      }
+      await onProgress?.(18, "Logged in");
+    } else {
+      await onProgress?.(18, "No auth needed");
+    }
+    if (didLogin) {
+      try {
+        const freshState = await playwrightBrowser.contexts()[0]?.storageState();
+        if (freshState) {
+          await AppBrain.saveCrawlConfig(opts.tenantId, opts.projectId, { storageState: JSON.stringify(freshState) });
+          logger.info("Fresh session state captured and saved after credential login");
+          await onProgress?.(19, "Session saved \u2014 will reuse on next crawl");
+        }
+      } catch (e) {
+        logger.warn({ err: e }, "Could not save refreshed session state \u2014 crawl will continue");
+      }
+      await waitForURLChange(page, opts.appUrl);
+    }
+    const origin = new URL(page.url()).origin;
+    const maxScreens = opts.maxScreens ?? Infinity;
+    const segmentSize = opts.segmentSize ?? 9999999;
+    const selective = !!(opts.selectiveUrls && opts.selectiveUrls.length > 0);
+    if (!selective) {
+      await AppBrain.prepareForRecrawl(opts.tenantId, opts.projectId, opts.jobId);
+      await AppBrain.clearFlows(opts.tenantId, opts.projectId);
+    }
+    const canonicalize = (raw) => {
+      try {
+        const u = new URL(raw);
+        u.hostname = u.hostname.replace(/^www\./, "");
+        [
+          "utm_source",
+          "utm_medium",
+          "utm_campaign",
+          "utm_term",
+          "utm_content",
+          "ref",
+          "source",
+          "_gl",
+          "fbclid",
+          "gclid",
+          "ts",
+          "timestamp",
+          "cache",
+          "cacheBust",
+          "cache_bust",
+          "_",
+          "nonce",
+          "session",
+          "sid",
+          // WooCommerce / e-commerce action params
+          "add-to-cart",
+          "add_to_wishlist",
+          "quantity",
+          "wc-ajax",
+          "orderby",
+          "min_price",
+          "max_price",
+          "removed_item",
+          "restore",
+          "remove_item",
+          "undo_item",
+          "coupon_code",
+          "apply_coupon",
+          "added-to-cart",
+          "wc_error",
+          "wc_notice",
+          "wc_notice_type",
+          "update_cart",
+          "proceed",
+          "_wpnonce",
+          "post_type",
+          "product_cat",
+          // WooCommerce facet/layered-nav filter params — each combination generates a unique URL
+          // but all render the same template; not stripping these causes infinite URL explosion
+          // (N filter options × M values = exponential queue growth)
+          "filter_color",
+          "filter_colour",
+          "filter_size",
+          "filter_brand",
+          "filter_category",
+          "filter_rating",
+          "filter_stock_status",
+          "filter_pa_color",
+          "filter_pa_size",
+          "filter_pa_brand",
+          "filter_pa_material",
+          "filter_pa_weight",
+          "filter_pa_style",
+          "filter_pa_gender",
+          "filter_pa_age",
+          "filter_pa_type",
+          "filter_pa_pattern",
+          "query_type_color",
+          "query_type_size",
+          "query_type_brand",
+          "query_type_pa_color",
+          "query_type_pa_size",
+          "query_type_pa_brand",
+          "in_stock",
+          "on_sale",
+          // Shopify facet filters
+          "sort_by",
+          "constraint",
+          "filter.p.tag",
+          "filter.v.price.gte",
+          "filter.v.price.lte",
+          "filter.p.m.global.color",
+          "filter.p.m.global.size"
+        ].forEach((p) => u.searchParams.delete(p));
+        for (const key of Array.from(u.searchParams.keys())) {
+          if (/^filter[._]|^query_type_/i.test(key)) u.searchParams.delete(key);
+        }
+        u.searchParams.sort();
+        u.pathname = u.pathname.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=\/|$)/gi, "/{id}").replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, "/{id}").replace(/\/\d{4,}(?=\/|$)/g, "/{id}").replace(/\/\d+-\d+(?=\/|$)/g, "/{id}").replace(/\/+$/, "") || "/";
+        if (!/^#!?\//.test(u.hash)) u.hash = "";
+        return u.toString();
+      } catch {
+        return raw;
+      }
+    };
+    const gitInfo = getGitInfo();
+    const visited = /* @__PURE__ */ new Map();
+    if (opts.skipUrls?.length) {
+      for (const u of opts.skipUrls) visited.set(canonicalize(u), "skipped");
+    }
+    let checkpointSuccessCount = 0;
+    let maxSeenTotal = 0;
+    const startUrl = page.url();
+    const resumeUrls = opts.resumeUrls ?? [];
+    const gqlIntrospectionPromise = (async () => {
+      const gqlEndpoints = guessGraphQLEndpoints(opts.appUrl);
+      for (const ep of gqlEndpoints) {
+        const result = await introspectGraphQL(ep, { timeoutMs: 5e3 }).catch(() => null);
+        if (result?.schema) {
+          logger.info({ endpoint: ep, typeCount: result.schema.types.length }, "[crawler] GraphQL schema discovered");
+          return result;
+        }
+      }
+      return null;
+    })();
+    const sitemapUrls = opts.useSitemap && !selective && resumeUrls.length === 0 ? await discoverSitemapUrls(opts.appUrl, opts.sitemapUrl) : [];
+    const filteredResumeUrls = resumeUrls.filter((u) => {
+      try {
+        return !isGeneratedDynamicPath(new URL(u).pathname);
+      } catch {
+        return true;
+      }
+    });
+    const seedUrls = selective ? opts.selectiveUrls : filteredResumeUrls.length > 0 ? filteredResumeUrls : [
+      startUrl,
+      // User-configured seed URLs added to queue; link discovery still runs from each.
+      ...(opts.startUrls ?? []).filter((u) => canonicalize(u) !== canonicalize(startUrl)),
+      ...sitemapUrls.filter((u) => canonicalize(u) !== canonicalize(startUrl))
+    ];
+    const queued = new Set(seedUrls.map(canonicalize));
+    const queue = new CrawlFrontier(opts.traversalStrategy ?? "BFS", seedUrls, opts.bestFirstKeywords, opts.urlFilter);
+    const deduplicator = new ContentDeduplicator(0.88);
+    const deadLinkTracker = new DeadLinkTracker();
+    const redirectRegistry = new RedirectRegistry();
+    if (!selective && resumeUrls.length === 0) {
+      try {
+        const priorMap = projectForProfile ?? await AppBrain.getProjectMap(opts.tenantId, opts.projectId);
+        const orphanReport = await AppBrain.getOrphanScreens(opts.tenantId, opts.projectId).catch(() => null);
+        const orphanIdSet = new Set(
+          (orphanReport?.screens ?? []).filter((s) => s.classification !== "REACHABLE").map((s) => s.id)
+        );
+        const priorScreens = (priorMap?.screens ?? []).filter((s) => s.status !== "PENDING_REMOVAL").filter((s) => {
+          const u = s.url;
+          if (!u) return false;
+          try {
+            const { origin: origin2, pathname } = new URL(u);
+            if (origin2 !== new URL(startUrl).origin) return false;
+            if (isGeneratedDynamicPath(pathname)) return false;
+            return true;
+          } catch {
+            return false;
+          }
+        }).filter((s) => !queued.has(canonicalize(s.url)));
+        let orphanSeedCount = 0;
+        for (const s of priorScreens) {
+          const u = s.url;
+          queued.add(canonicalize(u));
+          const isOrphan = orphanIdSet.has(s.id);
+          queue.push(u, isOrphan ? { parentElements: 0 } : void 0);
+          if (isOrphan) orphanSeedCount++;
+        }
+        if (priorScreens.length > 0) {
+          logger.info(
+            { count: priorScreens.length, orphanBoosted: orphanSeedCount },
+            "[crawler] re-seeded prior screen URLs from DB"
+          );
+        }
+      } catch (err) {
+        logger.warn({ err: String(err) }, "[crawler] non-fatal: could not load prior screen URLs for re-seeding");
+      }
+    }
+    if (opts.useSitemap !== false) {
+      const g11SitemapUrls = [];
+      const sitemapCandidates = [
+        `${opts.appUrl.replace(/\/$/, "")}/sitemap.xml`,
+        `${opts.appUrl.replace(/\/$/, "")}/sitemap_index.xml`,
+        `${opts.appUrl.replace(/\/$/, "")}/robots.txt`
+      ];
+      const LOGIN_REDIRECT_PATH = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
+      try {
+        for (const candidate of sitemapCandidates) {
+          try {
+            const resp = await fetch(candidate, { signal: AbortSignal.timeout(8e3) }).catch(() => null);
+            if (!resp?.ok) continue;
+            const text = await resp.text();
+            if (candidate.endsWith("robots.txt")) {
+              const sitemapLines = text.split("\n").filter((l) => l.toLowerCase().startsWith("sitemap:"));
+              for (const line of sitemapLines) {
+                const u = line.split(":").slice(1).join(":").trim();
+                if (u) sitemapCandidates.push(u);
+              }
+              continue;
+            }
+            const locs = text.match(/<loc>([^<]+)<\/loc>/gi) ?? [];
+            for (const loc of locs) {
+              const u = loc.replace(/<\/?loc>/gi, "").trim();
+              if (u.startsWith("http")) g11SitemapUrls.push(u);
+            }
+            if (g11SitemapUrls.length > 0) break;
+          } catch {
+          }
+        }
+        if (g11SitemapUrls.length > 0) {
+          logger.info({ count: g11SitemapUrls.length }, "[crawler] G11: sitemap discovered public URL candidates");
+          const sitemapOrigin = new URL(opts.appUrl).origin;
+          let sitemapCaptured = 0;
+          for (const sUrl of g11SitemapUrls.slice(0, 30)) {
+            try {
+              const u = new URL(sUrl);
+              if (u.origin !== sitemapOrigin) continue;
+              await page.goto(sUrl, { waitUntil: "domcontentloaded", timeout: 12e3 });
+              await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
+              });
+              await waitForStableDOM(page);
+              const landedUrl = page.url();
+              if (LOGIN_REDIRECT_PATH.test(new URL(landedUrl).pathname)) {
+                const canon = canonicalize(sUrl);
+                if (!queued.has(canon)) {
+                  queued.add(canon);
+                  queue.push(sUrl);
+                }
+                logger.info({ url: sUrl }, "[crawler] G11: sitemap URL requires auth \u2014 added to BFS queue");
+                continue;
+              }
+              const pagePath = u.pathname.replace(/^\//, "") || "home";
+              const pageName = pagePath.split("/").map((s) => s.charAt(0).toUpperCase() + s.slice(1)).join(" ") || "Home";
+              const shotPath = path.join(opts.screenshotDir ?? "/tmp", `sitemap-preauth-${sitemapCaptured}.png`);
+              await waitForScreenshotReady(page);
+              await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {
+              });
+              const pubScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, pageName, sUrl, opts.jobId, "SITEMAP").catch(() => null);
+              if (pubScreen) {
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, "DESKTOP").catch(() => {
+                });
+                logger.info({ url: sUrl, name: pageName }, "[crawler] G11: captured public sitemap page");
+              }
+              sitemapCaptured++;
+            } catch (sitemapPageErr) {
+              logger.warn({ url: sUrl, err: String(sitemapPageErr) }, "[crawler] G11: sitemap page failed \u2014 skipping");
+            }
+          }
+          logger.info({ captured: sitemapCaptured }, "[crawler] G11: pre-auth sitemap capture complete");
+          await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => {
+          });
+          await waitForStableDOM(page);
+        }
+      } catch (sitemapErr) {
+        logger.warn({ err: String(sitemapErr) }, "[crawler] G11: pre-auth sitemap discovery failed \u2014 continuing");
+      }
+    }
+    const canonicalEntryUrl = canonicalize(startUrl);
+    const canonicalSitemapUrls = new Set(sitemapUrls.map(canonicalize));
+    const canonicalSeedUrls = new Set(seedUrls.map(canonicalize));
+    const discoveryMethodFor = (url) => {
+      const c = canonicalize(url);
+      if (c === canonicalEntryUrl) return "ENTRY_POINT";
+      if (canonicalSitemapUrls.has(c)) return "SITEMAP";
+      if (canonicalSeedUrls.has(c)) return "SEED_URL";
+      return "LINK_FOLLOW";
+    };
+    const edges = [];
+    const pendingTransitions = [];
+    const discovered = [];
+    const performanceMetrics = [];
+    const seenDomHashes = new Set(opts.knownDomHashes ?? []);
+    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+    const segIdx = opts.segmentIndex ?? 1;
+    const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
+    let avgNavMs = 0;
+    let navCount = 0;
+    let aiErrorStreak = 0;
+    let aiSkipCount = 0;
+    let aiRateLimitWarned = false;
+    let totalAttemptsThisSegment = 0;
+    const maxAttempts = Math.max(segmentSize * 4, 200);
+    while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+      if (++totalAttemptsThisSegment > maxAttempts) {
+        logger.warn({ maxAttempts, visited: visited.size, queued: queue.length }, "[crawler] attempt cap reached \u2014 stopping to prevent URL explosion");
+        break;
+      }
+      if (opts.abortSignal?.aborted) {
+        logger.info("Abort signal received \u2014 stopping crawl");
+        throw new Error("CRAWL_ABORTED");
+      }
+      if (opts.autoThrottle && navCount > 0) {
+        const delayMs = Math.min(5e3, Math.max(0, avgNavMs * 0.5));
+        if (delayMs > 100) await page.waitForTimeout(delayMs).catch(() => {
+        });
+      }
+      const url = queue.shift();
+      const canonUrl = canonicalize(url);
+      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size + 1}/${segmentSize}` }, "[crawler] visiting URL");
+      const apiObservationStartIndex = capturedApiEndpoints.length;
+      if (visited.has(canonUrl)) continue;
+      if (robotsCache && !await robotsCache.isAllowed(url)) {
+        logger.info({ url }, "[crawler] robots.txt: skipping disallowed URL");
+        continue;
+      }
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) {
+          console.log(`[crawler] skip content page: ${url}`);
+          continue;
+        }
+        if (isGeneratedDynamicPath(pathname)) {
+          console.log(`[crawler] skip generated dynamic route: ${url}`);
+          continue;
+        }
+      } catch {
+      }
+      try {
+        let securityHeaders = null;
+        let httpStatus = null;
+        const urlBase = url.replace(/[?#].*/, "");
+        const cdpSession = page.mainSession ?? page._session ?? null;
+        const secHeadersCdp = (evt) => {
+          if (securityHeaders) return;
+          try {
+            const responseUrl = evt?.response?.url ?? "";
+            if (!responseUrl.startsWith(urlBase)) return;
+            httpStatus = evt?.response?.status ?? null;
+            const headers = {};
+            for (const h of evt?.response?.headers ?? []) headers[h.name.toLowerCase()] = h.value;
+            securityHeaders = {
+              csp: headers["content-security-policy"] ?? headers["content-security-policy-report-only"] ?? null,
+              hsts: headers["strict-transport-security"] ?? null,
+              xContentType: headers["x-content-type-options"] ?? null,
+              xFrame: headers["x-frame-options"] ?? null,
+              referrer: headers["referrer-policy"] ?? null
+            };
+          } catch {
+          }
+        };
+        if (cdpSession?.on) cdpSession.on("Network.responseReceived", secHeadersCdp);
+        const wsDetector = attachWebSocketDetector(cdpSession);
+        if (url !== page.url()) {
+          const navStart = Date.now();
+          const redirectHops = [];
+          const onRedirectResponse = (res) => {
+            const st = res.status();
+            if (st >= 300 && st < 400) {
+              redirectHops.push({ from: res.url(), to: res.headers()["location"] ?? "", statusCode: st });
+            } else if (st >= 400) {
+              deadLinkTracker.record(res.url(), st, { sourceUrl: url });
+            }
+          };
+          rawPage.on("response", onRedirectResponse);
+          await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch((navErr) => {
+            logger.warn({ url, err: navErr?.message ?? String(navErr) }, "[crawler] navigation failed \u2014 skipping");
+          });
+          rawPage.off("response", onRedirectResponse);
+          if (redirectHops.length > 0) {
+            redirectRegistry.register({
+              originalUrl: url,
+              finalUrl: page.url(),
+              hops: redirectHops.map((h, i) => ({ ...h, durationMs: i === 0 ? Date.now() - navStart : 0 })),
+              isCircular: new Set(redirectHops.map((h) => h.from)).size < redirectHops.length,
+              hopCount: redirectHops.length,
+              totalDurationMs: Date.now() - navStart
+            });
+          }
+          const navMs = Date.now() - navStart;
+          navCount++;
+          avgNavMs = avgNavMs + (navMs - avgNavMs) / navCount;
+        }
+        if (page.url().startsWith("chrome-error://")) {
+          console.warn(`[crawler] connection failed for ${url} \u2014 skipping (chrome-error page)`);
+          continue;
+        }
+        await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+        });
+        await waitForStableDOM(page);
+        if (!opts.discoveryMode) {
+          try {
+            const appModalVisible = await page.evaluate(() => {
+              const dialogs = Array.from(document.querySelectorAll('[role="dialog"], [role="alertdialog"], [aria-modal="true"]'));
+              return dialogs.some((el) => {
+                const text = (el.textContent ?? "").toLowerCase();
+                const isCookieBanner = /(cookie|consent|gdpr|privacy policy|accept all|we use cookies)/i.test(text);
+                const isVisible = el.offsetParent !== null || el.style.display !== "none";
+                return isVisible && !isCookieBanner;
+              });
+            }).catch(() => false);
+            if (appModalVisible) {
+              const modalHash = url.replace(/[^a-z0-9]/gi, "").slice(-12);
+              const modalShotPath = path.join(opts.screenshotDir ?? "/tmp", `modal-${modalHash}.png`);
+              await page.screenshot({ path: modalShotPath, fullPage: false }).catch(() => {
+              });
+              const modalName = await page.evaluate(() => {
+                const h = document.querySelector('[role="dialog"] h1,[role="dialog"] h2,[role="dialog"] [role="heading"]');
+                return h?.textContent?.trim() ?? "Modal";
+              }).catch(() => "Modal");
+              const modalScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, modalName, url, opts.jobId, discoveryMethodFor(url)).catch(() => null);
+              if (modalScreen) {
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, "DESKTOP").catch(() => {
+                });
+                logger.info({ url, name: modalName }, "[crawler] captured app modal before dismiss");
+              }
+            }
+          } catch {
+          }
+          await dismissOverlays(page);
+          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }).catch(() => {
+          });
+        }
+        if (cdpSession?.off) cdpSession.off("Network.responseReceived", secHeadersCdp);
+        wsDetector.detach();
+        if (httpStatus === 429) {
+          const backoffMs = Math.min(6e4, 1e3 * Math.pow(2, Math.min(navCount, 6)));
+          logger.warn({ url, httpStatus, backoffMs }, "[crawler] 429 rate-limited \u2014 backing off");
+          await new Promise((r) => setTimeout(r, backoffMs + Math.random() * 1e3));
+        }
+        if (httpStatus !== null && httpStatus >= 400) {
+          logger.info({ url, httpStatus }, "[crawler] skip \u2014 server returned HTTP error");
+          continue;
+        }
+        const BFS_ERROR_PATH = /\/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)(\/|$|\?)/i;
+        const BFS_LOGIN_PATH = /\/(login|signin|sign-in|auth|log-in|sso|account\/login)(\/|$|\?)/i;
+        const landedUrl = page.url();
+        const weWantedThisPage = BFS_ERROR_PATH.test(new URL(url).pathname) || BFS_LOGIN_PATH.test(new URL(url).pathname);
+        const redirectedToError = !weWantedThisPage && (BFS_ERROR_PATH.test(new URL(landedUrl).pathname) || BFS_LOGIN_PATH.test(new URL(landedUrl).pathname));
+        if (redirectedToError) {
+          logger.warn({ requested: url, landed: landedUrl }, "[crawler] Mid-crawl redirect to auth/error \u2014 attempting re-auth");
+          let reauthed = false;
+          if (opts.credentials) {
+            reauthed = await loginWithStagehand(
+              stagehand,
+              opts.credentials,
+              opts.appUrl,
+              { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider },
+              { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId },
+              void 0,
+              { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId }
+            ).catch(() => false);
+            if (reauthed) {
+              logger.info({ url }, "[crawler] Re-auth succeeded \u2014 retrying URL");
+              await page.goto(url, { waitUntil: "domcontentloaded" }).catch(() => {
+              });
+              await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+              });
+              await waitForStableDOM(page);
+              const afterReauthUrl = page.url();
+              if (BFS_ERROR_PATH.test(new URL(afterReauthUrl).pathname) || BFS_LOGIN_PATH.test(new URL(afterReauthUrl).pathname)) {
+                logger.warn({ url }, "[crawler] Still redirecting after re-auth \u2014 skipping this URL");
+                continue;
+              }
+            } else {
+              logger.warn({ url }, "[crawler] Re-auth failed \u2014 stopping BFS (session unrecoverable)");
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                type: "AUTH_FAILURE",
+                payload: { url, landed: landedUrl, reason: "Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl." }
+              }).catch(() => {
+              });
+              break;
+            }
+          } else {
+            logger.warn("[crawler] Session expired mid-crawl, no credentials to recover \u2014 stopping BFS");
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId,
+              type: "AUTH_FAILURE",
+              payload: { url, landed: landedUrl, reason: "Session expired and no credentials are configured. Add credentials in project settings and re-capture the session." }
+            }).catch(() => {
+            });
+            break;
+          }
+        }
+        if (!redirectedToError && landedUrl !== url) {
+          logger.info({ requested: url, landed: landedUrl }, "[crawler] navigated with redirect");
+        }
+        if (opts.discoveryMode) {
+          visited.set(canonicalize(url), url);
+          await (rawPage ?? page).waitForLoadState("networkidle", { timeout: 1e4 }).catch(() => {
+          });
+          await page.waitForTimeout(500).catch(() => {
+          });
+          const rawLinksSet = /* @__PURE__ */ new Set();
+          const addLink = (h) => {
+            if (!h || h === "#" || h.startsWith("javascript:") || h.startsWith("mailto:") || h.startsWith("tel:")) return;
+            try {
+              const href = new URL(h, origin).toString();
+              if (isUsefulNavigationUrl(href, origin)) rawLinksSet.add(href);
+            } catch {
+            }
+          };
+          const pwPage = rawPage ?? page;
+          await pwPage.locator("a[href]").evaluateAll(
+            (anchors) => anchors.map((a) => a.href || a.getAttribute("href") || "")
+          ).then((hrefs) => hrefs.forEach(addLink)).catch(() => {
+          });
+          await pwPage.locator("[data-href],[data-url],[data-link]").evaluateAll(
+            (els) => els.map((el) => el.dataset?.href || el.dataset?.url || el.dataset?.link || "")
+          ).then((hrefs) => hrefs.forEach(addLink)).catch(() => {
+          });
+          await pwPage.locator("[href]:not(a):not(link):not(base):not(script):not(style)").evaluateAll(
+            (els) => els.map((el) => el.getAttribute("href") || "")
+          ).then((hrefs) => hrefs.forEach(addLink)).catch(() => {
+          });
+          const playwrightCount = rawLinksSet.size;
+          console.log(`[discover] playwright-locator found ${playwrightCount} links for ${url}`);
+          const evalPage = rawPage ?? page;
+          await evalPage.evaluate((o) => {
+            try {
+              const hrefs = [];
+              const add = (h) => {
+                if (h && h !== "#" && !h.startsWith("javascript:") && !h.startsWith("mailto:") && !h.startsWith("tel:")) {
+                  if (h.startsWith(o) || h.startsWith("/")) hrefs.push(h);
+                }
+              };
+              const queue2 = [document];
+              const seenRoots = /* @__PURE__ */ new WeakSet();
+              seenRoots.add(document);
+              let shadowRootsFound = 0;
+              while (queue2.length > 0) {
+                const root = queue2.shift();
+                if (!root) continue;
+                try {
+                  root.querySelectorAll("a[href]").forEach((a) => add(a.href || a.getAttribute("href") || ""));
+                  root.querySelectorAll("[data-href],[data-url],[data-link]").forEach((el) => {
+                    add(el.dataset && (el.dataset.href || el.dataset.url || el.dataset.link) || "");
+                  });
+                  root.querySelectorAll("[href]:not(a):not(link):not(base):not(script):not(style)").forEach((el) => {
+                    const h = el.getAttribute("href") || "";
+                    if (h && !h.startsWith("#")) add(h);
+                  });
+                  root.querySelectorAll("*").forEach((el) => {
+                    try {
+                      if (el.shadowRoot && !seenRoots.has(el.shadowRoot)) {
+                        seenRoots.add(el.shadowRoot);
+                        queue2.push(el.shadowRoot);
+                        shadowRootsFound++;
+                      }
+                    } catch (_) {
+                    }
+                  });
+                } catch (_) {
+                }
+              }
+              return { hrefs, shadowRootsFound };
+            } catch {
+              return { hrefs: [], shadowRootsFound: 0 };
+            }
+          }, origin).then((result) => {
+            (result?.hrefs ?? []).forEach(addLink);
+            if (result?.shadowRootsFound > 0) {
+              console.log(`[discover] bfs found ${result.shadowRootsFound} shadow roots, added ${rawLinksSet.size - playwrightCount} extra links for ${url}`);
+            }
+          }).catch(() => {
+          });
+          await evalPage.evaluate(() => {
+            const routes = [];
+            try {
+              const nd = window.__NEXT_DATA__;
+              if (nd?.page) routes.push(nd.page);
+            } catch (_) {
+            }
+            try {
+              const w = window;
+              const rr = w.__reactRouterRoutes || w.__reactRouterManifest?.routes;
+              if (rr) {
+                const collect = (r) => {
+                  if (r?.path && r.path !== "*") routes.push(r.path.startsWith("/") ? r.path : "/" + r.path);
+                  if (r?.children) r.children.forEach(collect);
+                };
+                (Array.isArray(rr) ? rr : Object.values(rr)).forEach(collect);
+              }
+            } catch (_) {
+            }
+            return routes;
+          }).then((routes) => {
+            (routes ?? []).forEach((p) => {
+              try {
+                addLink(new URL(p, origin).href);
+              } catch {
+              }
+            });
+          }).catch(() => {
+          });
+          const rawLinks = Array.from(rawLinksSet);
+          console.log(`[discover] ${url} \u2192 rawLinks=${rawLinks.length} origin=${origin} pageUrl=${page.url()}`);
+          if (rawLinks.length > 0) console.log(`[discover] sample links: ${rawLinks.slice(0, 5).join(" | ")}`);
+          let newlyQueued = 0;
+          for (const link of rawLinks) {
+            try {
+              const resolved = link.startsWith("/") ? `${origin}${link}` : link;
+              const u = new URL(resolved);
+              if (!/^#!?\//.test(u.hash)) u.hash = "";
+              const clean = u.toString();
+              if (!isUsefulNavigationUrl(clean, origin)) continue;
+              const norm = canonicalize(clean);
+              if (!visited.has(norm) && !queued.has(norm)) {
+                queued.add(norm);
+                queue.push(clean);
+                newlyQueued++;
+              }
+            } catch {
+            }
+          }
+          if (newlyQueued > 0) console.log(`[discover] queued ${newlyQueued} new URLs, queue size=${queue.length}`);
+          const pct = 10 + Math.round(visited.size / Math.max(1, opts.maxScreens ?? 1e3) * 80);
+          await onProgress?.(Math.min(pct, 89), `Discovered ${visited.size} URLs\u2026`);
+          if (opts.onUrlsDiscovered) {
+            await opts.onUrlsDiscovered(Array.from(visited.values())).catch(() => {
+            });
+          }
+          continue;
+        }
+        const preDomHash = await computeDomHash(page);
+        if (seenDomHashes.has(preDomHash)) {
+          console.log(`[crawler] skip ${url} \u2014 duplicate DOM structure (same content as prior page)`);
+          continue;
+        }
+        const pageTitle = await page.title().catch(() => "");
+        if (/\b(not found|404|page not found|this page could not be found)\b/i.test(pageTitle)) {
+          console.log(`[crawler] skip ${url} \u2014 error page detected (title: "${pageTitle}")`);
+          seenDomHashes.add(preDomHash);
+          continue;
+        }
+        const rawPerf = await page.evaluate(() => {
+          const nav = performance.getEntriesByType("navigation")[0];
+          const paints = performance.getEntriesByType("paint");
+          const lcpFromObserver = window.__zetaLCP ?? null;
+          const lcpEntries = performance.getEntriesByType("largest-contentful-paint");
+          const lcpFromEntries = lcpEntries.length > 0 ? Math.round(lcpEntries[lcpEntries.length - 1].startTime) : void 0;
+          const stashedLcp = window.__zetaLCP ?? null;
+          const stashedCls = window.__zetaCLS;
+          return {
+            ttfb: nav ? Math.round(nav.responseStart - nav.fetchStart) : void 0,
+            fcp: paints.find((p) => p.name === "first-contentful-paint")?.startTime ? Math.round(paints.find((p) => p.name === "first-contentful-paint").startTime) : void 0,
+            lcp: lcpFromObserver != null ? Math.round(lcpFromObserver) : lcpFromEntries,
+            cls: typeof stashedCls === "number" ? Math.round(stashedCls * 1e3) / 1e3 : void 0
+          };
+        }).catch(() => null);
+        const a11yResult = await (async () => {
+          try {
+            const _require = createRequire(import.meta.url);
+            const axePath = _require.resolve("axe-core");
+            const axeScript = readFileSync(axePath, "utf8");
+            await page.addScriptTag({ content: axeScript });
+            const result = await page.evaluate(async () => {
+              return window.axe.run(document, {
+                runOnly: { type: "tag", values: ["wcag2a", "wcag2aa", "best-practice"] },
+                resultTypes: ["violations"]
+              });
+            });
+            const violations = result.violations.map((v) => ({
+              impact: v.impact,
+              rule: v.id,
+              description: v.description,
+              // First offending node's outer HTML (truncated later in saveA11yViolations)
+              nodeHtml: v.nodes?.[0]?.html ?? void 0,
+              wcagTags: v.tags.filter((t) => t.startsWith("wcag") || t.startsWith("best-"))
+            }));
+            const criticalCount = violations.filter((v) => v.impact === "critical").length;
+            return { violations, violationCount: violations.length, criticalCount };
+          } catch {
+            return null;
+          }
+        })();
+        const a11yScore = a11yResult ? Math.max(0, 100 - a11yResult.criticalCount * 20 - (a11yResult.violationCount - a11yResult.criticalCount) * 5) : null;
+        const name = await (async () => {
+          let brandName = "";
+          let pathName = "Home";
+          try {
+            const u = new URL(url);
+            const hostname = u.hostname.replace(/^www\./, "").split(".")[0].toLowerCase();
+            const allSegs = u.pathname.split("/").filter(Boolean);
+            const IS_ID = (s) => /^\d+$/.test(s) || // numeric
+            /^[0-9a-f]{8,}$/i.test(s) || // hex id
+            /^[a-z]{1,2}[0-9a-z]{20,}$/i.test(s);
+            const readable = allSegs.filter((s) => !IS_ID(s)).map((s) => s.charAt(0).toUpperCase() + s.slice(1).replace(/[-_]/g, " "));
+            if (readable.length > 0) {
+              const last = readable[readable.length - 1];
+              const GENERIC = ["edit", "new", "create", "view", "show", "index", "detail", "details"];
+              if (GENERIC.includes(last.toLowerCase()) && readable.length > 1) {
+                pathName = `${readable[readable.length - 2]} \xB7 ${last}`;
+              } else {
+                pathName = last;
+              }
+            } else if (allSegs.length > 0) {
+              const lastSeg = allSegs[allSegs.length - 1];
+              pathName = lastSeg.slice(0, 8).toUpperCase();
+            }
+            const SIG_PARAMS = ["product_cat", "category", "cat", "type", "tag", "brand", "filter", "tab", "section", "view"];
+            const paramSuffix = SIG_PARAMS.map((p) => {
+              const v = u.searchParams.get(p);
+              return v ? ` \xB7 ${v.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase())}` : "";
+            }).find(Boolean) ?? "";
+            pathName = pathName + paramSuffix;
+            brandName = hostname;
+          } catch {
+          }
+          try {
+            const title = await page.title();
+            if (title && title.length > 0 && title.length < 120) {
+              const parts = title.split(/\s*[|—\-–·]\s*/);
+              if (parts.length > 1) brandName = (parts[parts.length - 1] ?? brandName).trim().toLowerCase();
+              if (parts.length > 1) {
+                const candidate2 = parts[0].trim();
+                if (candidate2.length > 2 && !/^\d[\d\s]*$/.test(candidate2)) return candidate2;
+              }
+              const candidate = parts[0].trim();
+              const isBrand = candidate.split(/\s+/).length === 1 && (candidate.toLowerCase().includes(brandName) || candidate.toLowerCase() === brandName);
+              if (!isBrand && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
+                return candidate;
+              }
+            }
+          } catch {
+          }
+          try {
+            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: 2e3 }).catch(() => null);
+            if (activeNav) {
+              const t = activeNav.trim();
+              if (t.length > 1 && t.length < 60 && t.toLowerCase() !== brandName) return t;
+            }
+          } catch {
+          }
+          try {
+            const selectors = [
+              "main h1",
+              '[role="main"] h1',
+              ".page-title",
+              ".page-header h1",
+              ".content h1",
+              "main h2",
+              '[role="main"] h2',
+              ".page-header h2"
+            ];
+            for (const sel of selectors) {
+              const h = await page.locator(sel).first().textContent({ timeout: 2e3 }).catch(() => null);
+              if (!h) continue;
+              const t = h.trim();
+              if (t.length < 2 || t.length > 80) continue;
+              if (t.toLowerCase() === brandName) continue;
+              if (brandName && t.toLowerCase().replace(/\s/g, "").includes(brandName.replace(/\s/g, ""))) continue;
+              return t;
+            }
+          } catch {
+          }
+          try {
+            const hostname = new URL(url).hostname.replace(/^www\./, "").split(".")[0].toLowerCase();
+            if (pathName !== "Home" || name.toLowerCase() === hostname || name.toLowerCase().replace(/\s/g, "") === hostname) {
+              if (name.toLowerCase() === hostname || name.toLowerCase().replace(/\s/g, "") === hostname) {
+                return pathName;
+              }
+            }
+          } catch {
+          }
+          return pathName;
+        })();
+        const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
+        visited.set(canonicalize(url), screen.id);
+        const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${screen.id}-desktop-v${screen.version}.png`);
+        await (async () => {
+          const deadline = Date.now() + 4e3;
+          while (Date.now() < deadline) {
+            const ready = await page.evaluate(() => {
+              const busy = document.querySelectorAll('[aria-busy="true"], [data-loading="true"]');
+              const skeletons = document.querySelectorAll('.skeleton, [class*="skeleton"], [class*="Skeleton"]');
+              return busy.length === 0 && skeletons.length === 0;
+            }).catch(() => true);
+            if (ready) break;
+            await new Promise((r) => setTimeout(r, 200));
+          }
+        })();
+        void (async () => {
+          const screenId = screen.id;
+          const [seoR, contrastR, flagsR, langR, formsR] = await Promise.allSettled([
+            checkSeo(page, url),
+            auditColorContrast(page, url),
+            detectFeatureFlags(page, url),
+            detectLanguage(page, url),
+            detectForms(page)
+          ]);
+          const base = { screenId, jobId: opts.jobId };
+          if (seoR.status === "fulfilled")
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: "SEO_CHECK", payload: seoR.value }).catch(() => {
+            });
+          if (contrastR.status === "fulfilled" && contrastR.value.issues?.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: "COLOR_CONTRAST", payload: contrastR.value }).catch(() => {
+            });
+          if (flagsR.status === "fulfilled" && flagsR.value.flags?.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: "FEATURE_FLAGS", payload: flagsR.value }).catch(() => {
+            });
+          if (langR.status === "fulfilled")
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: "LANGUAGE_DETECTION", payload: langR.value }).catch(() => {
+            });
+          if (formsR.status === "fulfilled" && Array.isArray(formsR.value) && formsR.value.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: "FORMS_DETECTED", payload: { forms: formsR.value } }).catch(() => {
+            });
+        })();
+        try {
+          await page.addStyleTag({ content: "*, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; }" });
+        } catch {
+        }
+        const canonUrl2 = canonicalize(url);
+        const linkRecipe = await AppBrain.getActionRecipe(
+          opts.tenantId,
+          opts.projectId,
+          new URL(url).hostname,
+          `link-discovery:${canonUrl2}`
+        ).catch(() => null);
+        const cachedLinks = linkRecipe && linkRecipe.successRate >= 0.7 ? linkRecipe.steps.map((s) => s.href).filter((href) => !!href && isUsefulNavigationUrl(href, origin)) : null;
+        if (cachedLinks) {
+          logger.info({ url, cachedCount: cachedLinks.length }, "[crawler] link-discovery: using cached recipe \u2014 AI will still run and merge");
+        }
+        await onProgress?.(20, `Crawling ${visited.size} of ~${visited.size + queue.length} \xB7 ${(() => {
+          try {
+            return new URL(url).pathname || "/";
+          } catch {
+            return "/";
+          }
+        })()}`);
+        const preDomCount = await page.evaluate(
+          () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
+        ).catch(() => 0);
+        await waitForScreenshotReady(page);
+        const [, elements, nextLinks, atElements] = await Promise.all([
+          page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")),
+          extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []),
+          // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
+          // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
+          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0).then((freshLinks) => {
+            if (aiSkipCount > 0) aiSkipCount--;
+            aiErrorStreak = 0;
+            const merged = (cachedLinks ? Array.from(/* @__PURE__ */ new Set([...cachedLinks, ...freshLinks])) : freshLinks).filter((href) => isUsefulNavigationUrl(href, origin));
+            logger.info({ url, domLinks: freshLinks.length, cached: cachedLinks?.length ?? 0, merged: merged.length }, "[crawler] link-discovery: result");
+            if (merged.length > 0) {
+              AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
+                domain: new URL(url).hostname,
+                purpose: `link-discovery:${canonUrl2}`,
+                fingerprint: sha256(canonUrl2),
+                steps: merged.map((href) => ({ action: "link", href })),
+                successRate: 0.8
+              }).catch(() => {
+              });
+            }
+            return merged;
+          }).catch((err) => {
+            const fallback = err?.domLinks ?? [];
+            if (err?.isRateLimit) {
+              aiErrorStreak++;
+              if (aiErrorStreak >= 3) {
+                aiSkipCount = 10;
+                logger.warn("[crawler] 3 consecutive AI rate limits \u2014 skipping AI link discovery for next 10 screens");
+              }
+              if (!aiRateLimitWarned) {
+                aiRateLimitWarned = true;
+                onProgress?.(20, "AI quota exhausted \u2014 falling back to DOM-only link discovery").catch(() => {
+                });
+              }
+            }
+            return fallback;
+          }),
+          readScreen(page).catch(() => [])
+        ]);
+        logger.info({ url, screenId: screen.id, version: screen.version, elements: elements.length, links: nextLinks.length, atSample: atElements.slice(0, 3).map((a) => a.label) }, "[crawler] screen captured");
+        console.log("[crawler] AT:", JSON.stringify(atElements.slice(0, 5)));
+        try {
+          const domContent = await capturePageHtml(page);
+          if (!domContent) {
+            logger.warn({ url, screenId: screen.id }, "[crawler] markdown extraction skipped: DOM snapshot unavailable");
+          } else {
+            const domPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+            await fs.writeFile(domPath, domContent, "utf8");
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath).catch(() => {
+            });
+            const [iframeContent, shadowContent] = await Promise.all([
+              extractIframeContent(page).catch(() => []),
+              pierceShadowDom(page).catch(() => "")
+            ]);
+            const supplemental = [
+              ...iframeContent.map((f) => f.markdown).filter(Boolean),
+              shadowContent
+            ].join("\n\n").trim();
+            const extracted = extractMarkdown(
+              supplemental ? domContent + "\n<!-- iframe/shadow -->\n" + supplemental.slice(0, 3e3) : domContent,
+              url
+            );
+            let fitMarkdown;
+            if (extracted) {
+              const mdPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".md");
+              fitMarkdown = buildFitMarkdown(extracted.markdown, [
+                ...domainProfile.importantScreenPatterns,
+                ...domainProfile.workflowGoals,
+                ...opts.bestFirstKeywords ?? []
+              ]);
+              const { text: scrubbed, scrubbed: hadPii } = scrubPii(extracted.markdown);
+              if (hadPii) logger.info({ url }, "[crawler] PII detected and redacted from markdown");
+              const dedupResult = deduplicator.check(url, scrubbed);
+              if (dedupResult.isDuplicate) {
+                logger.info({ url, similarUrl: dedupResult.similarUrl, score: dedupResult.similarityScore }, "[crawler] near-duplicate page skipped");
+              }
+              await fs.writeFile(mdPath, scrubbed, "utf8");
+              const summaryPromise = (async () => {
+                try {
+                  const llmCfg = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => void 0);
+                  const snippet = extracted.markdown.slice(0, 1200);
+                  const { text, usage } = await askTracked(
+                    "You are a QA analyst. Given page content, write ONE crisp sentence describing what this screen does or shows. Be specific. No preamble.",
+                    `Screen: "${name}"
+URL: ${url}
+
+Content:
+${snippet}`,
+                    llmCfg,
+                    80
+                  );
+                  if (usage) {
+                    recordLLMCall({
+                      tenantId: opts.tenantId,
+                      projectId: opts.projectId,
+                      jobId: opts.jobId,
+                      agentType: "crawl_summary",
+                      label: `page_summary:${name}`,
+                      usage
+                    }).catch(() => {
+                    });
+                  }
+                  return text.trim().replace(/^["']|["']$/g, "");
+                } catch {
+                  return void 0;
+                }
+              })();
+              AppBrain.updateScreenMarkdown(opts.tenantId, opts.projectId, screen.id, {
+                markdownPath: mdPath,
+                markdownMethod: extracted.method,
+                markdownWordCount: extracted.wordCount,
+                markdown: fitMarkdown ?? extracted.markdown,
+                heading: extracted.title || name
+              }).catch((err) => console.warn("[crawler] markdown extraction save failed:", err));
+              summaryPromise.then((aiSummary) => {
+                if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => {
+                });
+              });
+            }
+            const structuredJson = summarizeStructuredElements(atElements);
+            const artifact = buildCrawlArtifactEnvelope({
+              tenantId: opts.tenantId,
+              projectId: opts.projectId,
+              jobId: opts.jobId,
+              url,
+              canonicalUrl: canonicalize(url),
+              screenId: screen.id,
+              screenshotPath: shotPath,
+              domPath,
+              domContent,
+              markdown: extracted?.markdown,
+              fitMarkdown,
+              accessibilityElements: atElements,
+              structuredJson,
+              evidence: [
+                { type: "domain_profile", payload: { industry: domainProfile.industry, confidence: domainProfile.confidence } },
+                { type: "engine_route", payload: routeDecision },
+                { type: "screen_fingerprints", payload: { urlCanonicalHash: sha256(canonicalize(url)) } }
+              ]
+            });
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              screenId: screen.id,
+              jobId: opts.jobId,
+              type: "SCREEN_ARTIFACT_ENVELOPE",
+              payload: artifact
+            }).catch((err) => console.warn("[crawler] artifact evidence save failed:", err?.message ?? err));
+          }
+        } catch (err) {
+          logger.warn({ url, screenId: screen.id, err: String(err?.message ?? err) }, "[crawler] markdown extraction failed");
+        }
+        if (a11yScore !== null) {
+          AppBrain.updateScreenA11y(opts.tenantId, screen.id, { a11yScore }).catch((err) => console.warn("[crawler] a11y score update failed:", err));
+        }
+        if (a11yResult !== null && a11yResult.violations.length > 0) {
+          AppBrain.saveA11yViolations(opts.tenantId, opts.projectId, screen.id, a11yResult.violations, opts.jobId).catch((err) => console.warn("[crawler] a11y violations save failed:", err));
+        }
+        const atByRole = /* @__PURE__ */ new Map();
+        for (const at of atElements) {
+          if (!atByRole.has(at.role)) atByRole.set(at.role, at.selectorHint);
+        }
+        AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, screen.id, "DESKTOP", shotPath, gitInfo.branch, gitInfo.commit || void 0).catch(() => {
+        });
+        const newElementHash = computeElementHash(elements);
+        AppBrain.updateElementHash(opts.tenantId, screen.id, newElementHash, screen.elementHash ?? null).catch((err) => console.warn("[crawler] element hash update failed:", err));
+        if (screen.version > 1) {
+          (async () => {
+            try {
+              const prevShot = await AppBrain.getPreviousScreenshotPath(opts.tenantId, screen.id, screen.version - 1);
+              if (prevShot) {
+                const { screenshotDiffPercent } = await import("./phash.js");
+                const { pHashDistance, changeLevel } = await screenshotDiffPercent(prevShot, shotPath);
+                await AppBrain.saveScreenDiff(opts.tenantId, opts.projectId, screen.id, screen.version - 1, screen.version, pHashDistance, changeLevel);
+              }
+            } catch (err) {
+              console.warn("[crawler] pHash diff skipped:", err);
+            }
+          })();
+        }
+        if (screen.version > 1) {
+          (async () => {
+            try {
+              const baseline = await AppBrain.getVisualBaselineForDiff(opts.tenantId, screen.id, "DESKTOP", gitInfo.branch);
+              if (baseline && baseline.storagePath !== shotPath) {
+                const { screenshotDiffPercent, gridLayoutDiff } = await import("./phash.js");
+                const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
+                  screenshotDiffPercent(baseline.storagePath, shotPath, baseline.ignoredRegions),
+                  gridLayoutDiff(baseline.storagePath, shotPath, baseline.ignoredRegions),
+                  AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, "DESKTOP")
+                ]);
+                let verdict = pHashVerdict(pHashDistance, changeLevel, [
+                  ...layoutChanges,
+                  ...elementShifts.map((s) => ({ location: "element", severity: s.severity, description: s.description }))
+                ]);
+                if (verdict.changeType === "SIGNIFICANT" || verdict.changeType === "CRITICAL") {
+                  try {
+                    const regions = baseline.ignoredRegions ?? [];
+                    const [baselineBuffer, newBuffer] = regions.length > 0 ? await (async () => {
+                      const { applyMask } = await import("./phash.js");
+                      return Promise.all([applyMask(baseline.storagePath, regions), applyMask(shotPath, regions)]);
+                    })() : await Promise.all([
+                      fs.readFile(baseline.storagePath),
+                      fs.readFile(shotPath)
+                    ]);
+                    const aiResult = await aiVisualCompare({
+                      baselineImageBase64: baselineBuffer.toString("base64"),
+                      newImageBase64: newBuffer.toString("base64"),
+                      screenName: screen.name,
+                      screenPurpose: "Application screen captured during automated crawl",
+                      tenantId: opts.tenantId,
+                      projectId: opts.projectId
+                    });
+                    verdict = mergeAiVisualVerdict(verdict, aiResult, screen.name);
+                  } catch (aiErr) {
+                    logger.warn({ screenId: screen.id, err: String(aiErr) }, "[crawler] AI visual compare failed, keeping pHash-only verdict");
+                  }
+                }
+                const { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+                if (changeType !== "NONE") {
+                  let newScreenshotCloudUrl;
+                  try {
+                    const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import("@qip/app-brain");
+                    const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
+                    if (storageCfg) {
+                      const key = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
+                      newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);
+                    }
+                  } catch {
+                  }
+                  await AppBrain.saveVisualDiff(opts.tenantId, {
+                    baselineId: baseline.id,
+                    newScreenshotPath: shotPath,
+                    newScreenshotCloudUrl,
+                    diffScore: pHashDistance,
+                    changeType,
+                    changes: allChanges,
+                    recommendation,
+                    bugReportDraft,
+                    aiSummary
+                  });
+                }
+              }
+            } catch (err) {
+              const missing = String(err?.message ?? err).includes("Input file is missing");
+              if (missing) {
+                logger.warn({ screenId: screen.id, version: screen.version, err: String(err?.message ?? err) }, "[crawler] VisualDiff skipped \u2014 baseline screenshot not on disk (stored in cloud only). Run a full recrawl to regenerate baselines.");
+              } else {
+                console.warn("[crawler] VisualDiff pipeline failed:", err);
+              }
+            }
+          })();
+        }
+        const linkMappings = await extractLinkMappings(page, origin);
+        for (const el of elements) {
+          const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark);
+          const label = el.meaning;
+          const role = el.role ?? "unknown";
+          const selectorHint = atByRole.get(role) ?? role;
+          const selectors = [
+            { type: "role-label", value: `${role}:${label}`, confidence: 0.9 },
+            { type: "aria-label", value: label, confidence: 0.85 },
+            { type: "tag-hint", value: selectorHint, confidence: 0.6 }
+          ];
+          AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+        }
+        for (const lm of linkMappings) {
+          const safety = assessActionSafety(lm.label, domainProfile);
+          if (!safety.allowed) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              screenId: screen.id,
+              jobId: opts.jobId,
+              type: "ACTION_BLOCKED_BY_SAFETY_POLICY",
+              payload: { label: lm.label, href: lm.href, decision: safety, profile: domainProfile.industry }
+            }).catch(() => {
+            });
+            continue;
+          }
+          pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
+          AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
+            domain: new URL(url).hostname,
+            purpose: `navigate:${lm.label}`,
+            fingerprint: sha256(`${canonicalize(url)}:${lm.label}:${canonicalize(lm.href)}`),
+            steps: [{ action: "click", label: lm.label, targetUrl: lm.href, fromUrl: url }],
+            successRate: 0.5
+          }).catch(() => {
+          });
+          if (!selective) {
+            try {
+              const targetPath = new URL(lm.href).pathname;
+              const canonLmHref = canonicalize(lm.href);
+              if (!SKIP_PATH.test(targetPath) && isUsefulNavigationUrl(lm.href, origin) && !queued.has(canonLmHref) && !visited.has(canonLmHref) && queued.size < maxScreens * 2) {
+                if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
+                queued.add(canonLmHref);
+                queue.push(lm.href, { parentElements: elements.length });
+                edges.push({ fromUrl: url, toUrl: lm.href });
+                logger.info({ fromUrl: url, newUrl: lm.href, label: lm.label, queueSize: queue.length }, "[crawler] anchor mapping added to queue");
+              }
+            } catch {
+            }
+          }
+        }
+        if (!selective) {
+          for (const href of nextLinks) {
+            edges.push({ fromUrl: url, toUrl: href });
+            try {
+              const hrefParams = new URL(href).searchParams;
+              const actionVal = hrefParams.get("action") ?? "";
+              if (/^yith-|^wc-|^vc_|^elementor/.test(actionVal)) continue;
+            } catch {
+            }
+            const canonHref = canonicalize(href);
+            if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 2) {
+              try {
+                const hrefPath = new URL(href).pathname;
+                if (isGeneratedDynamicPath(hrefPath)) continue;
+              } catch {
+              }
+              if (robotsCache && !await robotsCache.isAllowed(href)) continue;
+              queued.add(canonHref);
+              queue.push(href, { parentElements: elements.length });
+              logger.info({ newUrl: href, queueSize: queue.length }, "[crawler] new URL added to queue");
+            }
+          }
+          logger.info({ fromUrl: url, discovered: nextLinks.length, queueAfter: queue.length, visitedSoFar: visited.size }, "[crawler] link processing done");
+          try {
+            const jsRoutes = await page.evaluate(() => {
+              const routes = window.__zetaDiscoveredRoutes ?? [];
+              window.__zetaDiscoveredRoutes = [];
+              return routes;
+            }).catch(() => []);
+            for (const route of jsRoutes) {
+              try {
+                const abs = route.startsWith("http") ? route : `${origin}${route.startsWith("/") ? "" : "/"}${route}`;
+                const canonRoute = canonicalize(abs);
+                if (!queued.has(canonRoute) && !visited.has(canonRoute)) {
+                  queued.add(canonRoute);
+                  queue.push(abs);
+                  logger.info({ route: abs }, "[crawler] history.pushState: discovered SPA route");
+                }
+              } catch {
+              }
+            }
+          } catch {
+          }
+          if (preDomCount < 25 && !revisitQueue.includes(url)) {
+            try {
+              const postDomCount = await page.evaluate(
+                () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
+              ).catch(() => 0);
+              if (postDomCount > preDomCount * 1.3 + 4) {
+                revisitQueue.push(url);
+                logger.info({ url, preDomCount, postDomCount }, "[P3a] Async content detected \u2014 scheduled for re-visit");
+              }
+            } catch {
+            }
+          }
+          if (visited.size <= 5) {
+            try {
+              const jsDetectedRoutes = await detectJsRoutes(page, origin);
+              for (const route of jsDetectedRoutes) {
+                const canonRoute = canonicalize(route);
+                if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 2) {
+                  queued.add(canonRoute);
+                  queue.push(route, { parentElements: elements.length });
+                  logger.info({ route }, "[P2] JS router: discovered route added to queue");
+                }
+              }
+            } catch {
+            }
+          }
+          for (const relPath of [...xhrDiscoveredPaths]) {
+            xhrDiscoveredPaths.delete(relPath);
+            try {
+              const abs = `${origin}${relPath}`;
+              const canon = canonicalize(abs);
+              if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                queued.add(canon);
+                queue.push(abs);
+                logger.info({ url: abs }, "[P4] XHR: page URL from JSON response added to queue");
+              }
+            } catch {
+            }
+          }
+          if (nextLinks.length === 0 && elements.length < 5 && opts.skyvernConfig) {
+            try {
+              logger.info({ url }, "Gap 5: thin page detected \u2014 asking Skyvern to discover navigation URLs");
+              const skyvernNav = await skyvernNavigate(
+                url,
+                ["Find and return all navigation links, menu items, and page URLs accessible from this screen"],
+                opts.skyvernConfig
+              );
+              for (const skyvernUrl of skyvernNav.visitedUrls) {
+                const canonSkyvernUrl = canonicalize(skyvernUrl);
+                if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 2) {
+                  logger.info({ url: skyvernUrl }, "Gap 5: Skyvern-discovered URL added to ZeTa crawl queue");
+                  queued.add(canonSkyvernUrl);
+                  queue.push(skyvernUrl);
+                  pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: "Skyvern-discovered link", targetUrl: skyvernUrl });
+                }
+              }
+            } catch {
+            }
+          }
+        }
+        if (rawPerf) {
+          performanceMetrics.push({ url, screenName: name, ...rawPerf });
+        }
+        const screenApiEndpoints = capturedApiEndpoints.slice(apiObservationStartIndex);
+        if (screenApiEndpoints.length > 0) {
+          AppBrain.saveApiEndpointObservations(opts.tenantId, opts.projectId, screen.id, screenApiEndpoints).then(() => AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            screenId: screen.id,
+            jobId: opts.jobId,
+            type: "API_ENDPOINT_OBSERVATIONS",
+            payload: { endpoints: screenApiEndpoints }
+          })).catch((err) => console.warn("[crawler] API endpoint observation save failed:", err?.message ?? err));
+        }
+        if (opts.jobId && (rawPerf || process.env.ENABLE_LIGHTHOUSE)) {
+          (async () => {
+            try {
+              const lhScores = await runLighthouse(url);
+              await AppBrain.savePerformanceSnapshot(opts.tenantId, screen.id, opts.jobId, {
+                lcp: rawPerf?.lcp,
+                fcp: rawPerf?.fcp,
+                cls: rawPerf?.cls,
+                ttfb: rawPerf?.ttfb,
+                viewport: "DESKTOP",
+                ...lhScores ? {
+                  lighthousePerf: lhScores.perf,
+                  lighthouseA11y: lhScores.a11y,
+                  lighthouseSEO: lhScores.seo,
+                  lighthouseBP: lhScores.bp,
+                  lighthouseReport: lhScores.report
+                } : {}
+              });
+            } catch (perfErr) {
+              console.warn("[crawler] savePerformanceSnapshot failed (non-fatal):", perfErr);
+            }
+          })();
+        }
+        discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+        maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
+        const screenPct = Math.min(60, visited.size / Math.max(1, maxSeenTotal) * 60);
+        await onProgress?.(20 + Math.round(screenPct), `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
+        logger.info({ name, elements: elements.length }, "Screen crawled");
+        if (++checkpointSuccessCount % 10 === 0) {
+          await opts.onCheckpoint?.(
+            Array.from(visited.keys()),
+            queue.filter((u) => !visited.has(canonicalize(u)))
+          );
+        }
+      } catch (err) {
+        logger.warn({ url, err: err?.message ?? String(err), stack: err?.stack?.split("\n").slice(0, 5).join(" | ") }, "Screen crawl failed");
+        discovered.push({ url, error: err?.message ?? String(err) });
+      }
+    }
+    if (opts.discoveryMode) {
+      return {
+        discoveredUrls: Array.from(visited.values()),
+        screensFound: 0,
+        screensErrored: 0,
+        screensSkippedToNextSegment: 0,
+        isPartial: false,
+        discovered: [],
+        apiEndpoints: [],
+        performanceMetrics: [],
+        pendingUrls: [],
+        segmentIndex: segIdx,
+        hasMoreSegments: false,
+        domHashes: Array.from(seenDomHashes)
+      };
+    }
+    const successfulScreenCount = discovered.filter((d) => d.screenId).length;
+    if (successfulScreenCount < 10 && opts.skyvernConfig) {
+      await onProgress?.(78, "Few screens found \u2014 using Skyvern to discover additional navigation URLs");
+      const currentUrl = page.url();
+      const skyvernNav = await skyvernNavigate(
+        currentUrl,
+        [
+          "Explore the main navigation and visit each top-level section",
+          "Click any sidebar or top menu items to discover sub-pages"
+        ],
+        opts.skyvernConfig
+      );
+      const skyvernUrls = skyvernNav.visitedUrls.filter((u) => !visited.has(canonicalize(u)));
+      if (skyvernUrls.length > 0) {
+        logger.info({ count: skyvernUrls.length }, "Gap 5: Skyvern found new URLs \u2014 processing with ZeTa native crawler");
+        for (const skyvernUrl of skyvernUrls) {
+          if (visited.size >= maxScreens) break;
+          try {
+            await page.goto(skyvernUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
+            await page.waitForTimeout(1500);
+            const skyvernName = new URL(skyvernUrl).pathname.split("/").filter(Boolean).pop() ?? "Screen";
+            const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, skyvernName, skyvernUrl, opts.jobId, "SKYVERN_AI");
+            const shotPath = `${opts.screenshotDir ?? "/tmp"}/${screen.id}-v${screen.version}.png`;
+            await waitForScreenshotReady(page);
+            await page.screenshot({ path: shotPath, fullPage: true });
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP");
+            const [elems, links, linkMappings] = await Promise.all([
+              extractElements(stagehand, page, elementLLM ?? void 0),
+              discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0),
+              extractLinkMappings(page, origin)
+            ]);
+            for (const el of elems) {
+              await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+              });
+            }
+            visited.set(canonicalize(skyvernUrl), screen.id);
+            discovered.push({ url: skyvernUrl, screenId: screen.id, elements: elems.length, version: screen.version });
+            for (const lm of linkMappings) {
+              pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
+            }
+            for (const href of links) {
+              const c = canonicalize(href);
+              if (isUsefulNavigationUrl(href, origin) && !queued.has(c) && !visited.has(c)) {
+                queued.add(c);
+                queue.push(href);
+              }
+            }
+          } catch (err) {
+            logger.warn({ url: skyvernUrl, err: String(err) }, "Gap 5: ZeTa crawl of Skyvern URL failed");
+          }
+        }
+        while (queue.length > 0 && visited.size < maxScreens) {
+          const href = queue.shift();
+          const canonHref = canonicalize(href);
+          if (visited.has(canonHref)) continue;
+          if (!isUsefulNavigationUrl(href, origin)) continue;
+          try {
+            await page.goto(href, { waitUntil: "domcontentloaded", timeout: 2e4 });
+            await page.waitForTimeout(1e3);
+            const nm = new URL(href).pathname.split("/").filter(Boolean).pop() ?? "Screen";
+            const sc = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, "SKYVERN_AI");
+            const sp = `${opts.screenshotDir ?? "/tmp"}/${sc.id}-v${sc.version}.png`;
+            await waitForScreenshotReady(page);
+            await page.screenshot({ path: sp, fullPage: true });
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, "DESKTOP");
+            const [elems2, linkMappings2] = await Promise.all([
+              extractElements(stagehand, page, elementLLM ?? void 0),
+              extractLinkMappings(page, origin)
+            ]);
+            for (const el of elems2) {
+              await AppBrain.saveElement(opts.tenantId, opts.projectId, sc.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+              });
+            }
+            visited.set(canonHref, sc.id);
+            discovered.push({ url: href, screenId: sc.id, elements: elems2.length, version: sc.version });
+            for (const lm of linkMappings2) {
+              pendingTransitions.push({ fromScreenId: sc.id, elementMeaning: lm.label, targetUrl: lm.href });
+            }
+          } catch {
+          }
+        }
+      }
+    }
+    try {
+      const { runUrlPatternCompletion } = await import("./url-pattern-completer.js");
+      const llmCfgForPatterns = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => null);
+      const urlPatternCandidates = await runUrlPatternCompletion(
+        Array.from(visited.keys()),
+        opts.appUrl,
+        llmCfgForPatterns
+      );
+      if (urlPatternCandidates.length > 0) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts.jobId,
+          type: "URL_CANDIDATES",
+          payload: {
+            candidates: urlPatternCandidates,
+            hits: urlPatternCandidates.filter((c) => c.status === "hit").length,
+            total: urlPatternCandidates.length
+          }
+        }).catch(() => {
+        });
+        logger.info(
+          { hits: urlPatternCandidates.filter((c) => c.status === "hit").length, total: urlPatternCandidates.length },
+          "[P1] URL pattern completion saved to evidence"
+        );
+      }
+    } catch (err) {
+      logger.warn({ err: String(err) }, "[P1] URL pattern completion failed \u2014 non-fatal, continuing");
+    }
+    if (revisitQueue.length > 0) {
+      await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content\u2026`);
+      logger.info({ count: revisitQueue.length }, "[P3a] Starting async content re-visit pass");
+      for (const revisitUrl of revisitQueue.slice(0, 10)) {
+        const existingScreenId = visited.get(canonicalize(revisitUrl));
+        if (!existingScreenId) continue;
+        try {
+          await page.goto(revisitUrl, { waitUntil: "networkidle", timeout: 25e3 });
+          await page.waitForTimeout(2e3);
+          const shotPath2 = `${opts.screenshotDir ?? "/tmp"}/${existingScreenId}-revisit.png`;
+          await waitForScreenshotReady(page);
+          await page.screenshot({ path: shotPath2, fullPage: true });
+          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, "DESKTOP").catch(() => {
+          });
+          const revisitElements = await extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []);
+          for (const el of revisitElements) {
+            await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+            });
+          }
+          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false).catch(() => []);
+          for (const href of revisitLinks) {
+            const canon = canonicalize(href);
+            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+              queued.add(canon);
+              queue.push(href);
+            }
+          }
+          logger.info({ url: revisitUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, "[P3a] Re-visit complete");
+        } catch (err) {
+          logger.warn({ url: revisitUrl, err: String(err) }, "[P3a] Re-visit failed \u2014 non-fatal");
+        }
+      }
+    }
+    if (visited.size >= 3) {
+      try {
+        await onProgress?.(80, "Running gap analysis\u2026");
+        const llmCfgForGap = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => null);
+        const discoveredForGap = Array.from(visited.entries()).map(([url, screenId]) => {
+          const d = discovered.find((dd) => dd.screenId === screenId);
+          return { url, name: d ? new URL(url).pathname.split("/").filter(Boolean).pop() ?? "screen" : void 0, elements: d?.elements };
+        });
+        const gapCandidates = await runGapAnalysis(discoveredForGap, opts.appUrl, llmCfgForGap);
+        const confirmedGaps = gapCandidates.filter((g) => g.confirmedUrl && !visited.has(canonicalize(g.confirmedUrl)));
+        if (confirmedGaps.length > 0) {
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "URL_CANDIDATES",
+            payload: {
+              source: "gap-pass",
+              candidates: gapCandidates.map((g) => ({ url: g.confirmedUrl ?? g.urlPatterns[0], confidence: g.confidence, reason: g.rationale, status: g.confirmedUrl ? "hit" : "miss" })),
+              hits: confirmedGaps.length,
+              total: gapCandidates.length
+            }
+          }).catch(() => {
+          });
+          logger.info({ confirmed: confirmedGaps.length, total: gapCandidates.length }, "[P6] Gap pass: visiting confirmed gap URLs");
+          const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
+          for (const gap of confirmedGaps.slice(0, 8)) {
+            const gapUrl = gap.confirmedUrl;
+            if (visited.size >= maxScreens) break;
+            try {
+              await page.goto(gapUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
+              if (gapLoginPattern.test(page.url())) {
+                logger.warn({ gapUrl, landed: page.url() }, "[P6] Gap URL redirected to login \u2014 skipping");
+                continue;
+              }
+              await page.waitForTimeout(1500);
+              const gapName = gap.category;
+              const gapScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, gapName, gapUrl, opts.jobId, "ENTRY_POINT");
+              const gapShot = `${opts.screenshotDir ?? "/tmp"}/${gapScreen.id}-v${gapScreen.version}.png`;
+              await waitForScreenshotReady(page);
+              await page.screenshot({ path: gapShot, fullPage: true });
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, "DESKTOP");
+              const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
+                extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []),
+                discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false).catch(() => []),
+                extractLinkMappings(page, origin).catch(() => [])
+              ]);
+              for (const el of gapElems) {
+                await AppBrain.saveElement(opts.tenantId, opts.projectId, gapScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+                });
+              }
+              visited.set(canonicalize(gapUrl), gapScreen.id);
+              discovered.push({ url: gapUrl, screenId: gapScreen.id, elements: gapElems.length, version: gapScreen.version });
+              for (const lm of gapLinkMaps) {
+                pendingTransitions.push({ fromScreenId: gapScreen.id, elementMeaning: lm.label, targetUrl: lm.href });
+              }
+              for (const href of gapLinks) {
+                const canon = canonicalize(href);
+                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                  queued.add(canon);
+                  queue.push(href);
+                }
+              }
+              logger.info({ url: gapUrl, elements: gapElems.length, category: gap.category }, "[P6] Gap URL captured");
+            } catch (err) {
+              logger.warn({ url: gapUrl, err: String(err) }, "[P6] Gap URL visit failed \u2014 non-fatal");
+            }
+          }
+        } else {
+          logger.info({ total: gapCandidates.length }, "[P6] Gap pass: no confirmed gaps found");
+        }
+      } catch (err) {
+        logger.warn({ err: String(err) }, "[P6] Gap pass failed \u2014 non-fatal, continuing");
+      }
+    }
+    if (!selective && visited.size > 0) {
+      const discoveredUrls = Array.from(visited.keys());
+      await AppBrain.markRemovedScreens(opts.tenantId, opts.projectId, discoveredUrls, opts.jobId);
+    }
+    await onProgress?.(82, "Saving app map");
+    for (const edge of edges) {
+      const fromId = visited.get(canonicalize(edge.fromUrl));
+      const toId = visited.get(canonicalize(edge.toUrl));
+      if (fromId && toId && fromId !== toId) {
+        await AppBrain.saveFlow(opts.tenantId, opts.projectId, `${edge.fromUrl} -> ${edge.toUrl}`, fromId, toId);
+      }
+    }
+    await onProgress?.(84, "Wiring navigation graph (KAG)");
+    const visitedByPath = /* @__PURE__ */ new Map();
+    for (const [canonUrl] of visited) {
+      try {
+        const p = new URL(canonUrl).pathname;
+        const arr = visitedByPath.get(p);
+        if (arr) arr.push(canonUrl);
+        else visitedByPath.set(p, [canonUrl]);
+      } catch {
+      }
+    }
+    for (const pt of pendingTransitions) {
+      let toScreenId = visited.get(canonicalize(pt.targetUrl));
+      if (!toScreenId) {
+        try {
+          const targetCanon = new URL(canonicalize(pt.targetUrl));
+          const candidates = visitedByPath.get(targetCanon.pathname) ?? [];
+          for (const cand of candidates) {
+            const candUrl = new URL(cand);
+            let allMatch = true;
+            for (const [k, v] of targetCanon.searchParams) {
+              if (candUrl.searchParams.get(k) !== v) {
+                allMatch = false;
+                break;
+              }
+            }
+            if (allMatch) {
+              toScreenId = visited.get(cand);
+              break;
+            }
+          }
+        } catch {
+        }
+      }
+      if (toScreenId && toScreenId !== pt.fromScreenId) {
+        await AppBrain.setElementTransition(opts.tenantId, pt.fromScreenId, pt.elementMeaning, toScreenId).catch(() => {
+        });
+        await AppBrain.saveScreenTransition(opts.tenantId, opts.projectId, {
+          fromScreenId: pt.fromScreenId,
+          toScreenId,
+          action: {
+            type: "click",
+            label: pt.elementMeaning,
+            targetUrl: pt.targetUrl,
+            source: "crawl-link-mapping"
+          },
+          confidence: 0.8,
+          jobId: opts.jobId
+        }).catch(() => {
+        });
+      }
+    }
+    const transitionCount = pendingTransitions.filter((pt) => {
+      if (visited.get(canonicalize(pt.targetUrl))) return true;
+      try {
+        const tc = new URL(canonicalize(pt.targetUrl));
+        return (visitedByPath.get(tc.pathname) ?? []).some((cand) => {
+          const cu = new URL(cand);
+          for (const [k, v] of tc.searchParams) {
+            if (cu.searchParams.get(k) !== v) return false;
+          }
+          return true;
+        });
+      } catch {
+        return false;
+      }
+    }).length;
+    logger.info({ transitionCount }, "KAG: wired element\u2192screen transitions");
+    AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+      jobId: opts.jobId,
+      type: "WORKFLOW_GRAPH_SUMMARY",
+      payload: {
+        transitionCount,
+        screenCount: visited.size,
+        profile: domainProfile.industry
+      }
+    }).catch(() => {
+    });
+    if (capturedApiEndpoints.length > 0) {
+      logger.info({ count: capturedApiEndpoints.length }, "Captured unique API endpoint patterns");
+    }
+    if (visited.size >= 5) {
+      await onProgress?.(90, `App map saved \xB7 ${visited.size} screens, ${edges.length} flows wired`);
+    }
+    let sessionAliveForViewports = true;
+    try {
+      await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 15e3 });
+      const sessionCheckUrl = page.url();
+      const loginKeywords = /login|signin|sign-in|\/auth[/?#]/i;
+      if (loginKeywords.test(sessionCheckUrl)) {
+        logger.warn({ landed: sessionCheckUrl }, "[P5] Session expired before viewport pass \u2014 attempting re-auth");
+        let reauthed = false;
+        if (opts.credentials) {
+          reauthed = await loginWithStagehand(
+            stagehand,
+            opts.credentials,
+            opts.appUrl,
+            { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider },
+            { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId },
+            void 0,
+            { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId }
+          ).catch(() => false);
+        }
+        if (reauthed) {
+          logger.info("[P5] Re-auth succeeded \u2014 proceeding with viewport pass");
+        } else {
+          sessionAliveForViewports = false;
+          logger.warn({ landed: sessionCheckUrl }, "[P5] Re-auth failed \u2014 skipping mobile/tablet screenshots");
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "AUTH_FAILURE",
+            payload: {
+              url: opts.appUrl,
+              landed: sessionCheckUrl,
+              reason: "Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings."
+            }
+          }).catch(() => {
+          });
+        }
+      } else {
+        logger.info({ url: sessionCheckUrl }, "[P5] Session alive \u2014 proceeding with viewport pass");
+      }
+    } catch (err) {
+      logger.warn({ err: String(err) }, "[P5] Session health check failed \u2014 skipping viewport pass to avoid bad screenshots");
+      sessionAliveForViewports = false;
+    }
+    const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== "DESKTOP");
+    if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
+      const { devices, chromium: pwChromium } = await import("playwright");
+      const DEVICE_MAP = {
+        // Override viewport heights: Playwright's device descriptors use viewport-minus-browser-chrome
+        // (659 for iPhone 15, ~980 for iPad Pro 11). Override to full logical screen dimensions so
+        // screenshots match what users see on device and mobile-detection scripts pass.
+        MOBILE_PORTRAIT: { ...devices["iPhone 15"], viewport: { width: 393, height: 852 } },
+        TABLET: { ...devices["iPad Pro 11"], viewport: { width: 834, height: 1194 } }
+      };
+      let mainStorageState;
+      try {
+        mainStorageState = await playwrightBrowser.contexts()[0]?.storageState();
+      } catch {
+      }
+      for (const profile of extraProfiles) {
+        const deviceConfig = DEVICE_MAP[profile];
+        if (!deviceConfig) continue;
+        await onProgress?.(95, `Capturing ${profile} screenshots + elements\u2026`);
+        let mobileBrowser = null;
+        try {
+          mobileBrowser = await pwChromium.launch({
+            headless: true,
+            executablePath,
+            args: chromeArgs
+          });
+          const mobileCtx = await mobileBrowser.newContext({
+            ...deviceConfig,
+            ignoreHTTPSErrors: true,
+            ...mainStorageState ? { storageState: mainStorageState } : {}
+          });
+          const mobilePage = await mobileCtx.newPage();
+          for (const disc of discovered) {
+            if (!disc.screenId) continue;
+            try {
+              await mobilePage.goto(disc.url, { waitUntil: "domcontentloaded", timeout: 2e4 });
+              await mobilePage.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+              });
+              await dismissOverlays(mobilePage);
+              try {
+                await mobilePage.addStyleTag({ content: "*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }" });
+              } catch {
+              }
+              const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${disc.screenId}-${profile.toLowerCase()}-v${disc.version}.png`);
+              await waitForScreenshotReady(mobilePage);
+              const [, vpElements] = await Promise.all([
+                mobilePage.screenshot({ path: shotPath, fullPage: true }),
+                domFallbackElements(mobilePage)
+              ]);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version, profile);
+              AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, disc.screenId, profile, shotPath, gitInfo.branch, gitInfo.commit || void 0).catch(() => {
+              });
+              for (const el of vpElements) {
+                AppBrain.saveElement(
+                  opts.tenantId,
+                  opts.projectId,
+                  disc.screenId,
+                  el.meaning,
+                  el.role,
+                  el.expectedData,
+                  el.notes,
+                  void 0,
+                  true,
+                  el.boundingRect,
+                  el.ariaState,
+                  el.parentLandmark,
+                  profile
+                ).catch(() => {
+                });
+              }
+            } catch (err) {
+              console.warn(`[crawler] ${profile} pass failed for ${disc.url}:`, err);
+            }
+          }
+          await mobilePage.close().catch(() => {
+          });
+          await mobileCtx.close().catch(() => {
+          });
+        } catch (profileErr) {
+          console.warn(`[crawler] ${profile} browser launch failed:`, profileErr);
+        } finally {
+          await mobileBrowser?.close().catch(() => {
+          });
+        }
+      }
+    }
+    const shouldRunInteractiveProbe = opts.interactive || discovered.filter((d) => d.screenId).length < 30;
+    if (shouldRunInteractiveProbe) {
+      await onProgress?.(88, "Running interactive probe (capturing real errors & modals)");
+      const probeInputs = discovered.filter((d) => d.screenId).map((d) => ({ url: d.url, screenId: d.screenId, elements: d.elementFacts ?? [] }));
+      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
+    }
+    await onProgress?.(97, "Running post-crawl analysis");
+    const successScreens = discovered.filter((d) => !!d.screenId);
+    ;
+    (async () => {
+      try {
+        if (successScreens.length > 0) {
+          const sitemapUrls2 = screensToSitemapUrls(successScreens.map((d) => ({ url: d.url })));
+          const sitemapPath = path.join(opts.screenshotDir ?? "/tmp", "sitemap.xml");
+          await writeSitemap(sitemapUrls2, sitemapPath);
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "SITEMAP_WRITTEN",
+            payload: { path: sitemapPath, urlCount: sitemapUrls2.length }
+          }).catch(() => {
+          });
+        }
+        if (capturedApiEndpoints.length > 0) {
+          const spec = generateOpenApiSpec(capturedApiEndpoints, { title: `${opts.projectId} API`, version: "1.0.0" });
+          const yamlStr = specToYaml(spec);
+          const specPath = path.join(opts.screenshotDir ?? "/tmp", "openapi.yaml");
+          await fs.writeFile(specPath, yamlStr, "utf8");
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "OPENAPI_SPEC",
+            payload: { path: specPath, endpointCount: capturedApiEndpoints.length }
+          }).catch(() => {
+          });
+        }
+        const deadStats = deadLinkTracker.stats();
+        if (deadStats.total > 0) {
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "DEAD_LINKS",
+            payload: { ...deadStats, links: deadLinkTracker.getDeadLinks() }
+          }).catch(() => {
+          });
+        }
+        const redirectStats = redirectRegistry.stats();
+        if (redirectStats.withRedirects > 0) {
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "REDIRECT_CHAINS",
+            payload: { ...redirectStats, problematic: redirectRegistry.getProblematic() }
+          }).catch(() => {
+          });
+        }
+        if (successScreens.length > 0) {
+          const sampleScreens = successScreens.slice(0, 5).map((d) => ({
+            id: d.screenId,
+            url: d.url,
+            name: d.url
+          }));
+          const playwrightScript = generatePlaywrightTest(sampleScreens);
+          const testPath = path.join(opts.screenshotDir ?? "/tmp", "generated.spec.ts");
+          await fs.writeFile(testPath, playwrightScript, "utf8");
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            jobId: opts.jobId,
+            type: "PLAYWRIGHT_TEST_GENERATED",
+            payload: { path: testPath, screenCount: sampleScreens.length }
+          }).catch(() => {
+          });
+        }
+        const webhookUrl = opts.webhookUrl;
+        if (webhookUrl) {
+          const notifier = new WebhookNotifier({ url: webhookUrl, secret: opts.webhookSecret });
+          await notifier.notify("crawl.completed", opts.jobId ?? "", opts.projectId, opts.tenantId, {
+            screensFound: successScreens.length,
+            apiEndpoints: capturedApiEndpoints.length,
+            deadLinks: deadLinkTracker.stats().total
+          }).catch(() => {
+          });
+        }
+      } catch {
+      }
+    })();
+    await onProgress?.(98, "Finalizing");
+    const pendingUrls = queue.filter((u) => !visited.has(canonicalize(u)));
+    const hasMoreSegments = pendingUrls.length > 0;
+    const screensSucceeded = discovered.filter((d) => !!d.screenId).length;
+    const screensErrored = discovered.filter((d) => !!d.error && !d.screenId).length;
+    return {
+      screensFound: screensSucceeded,
+      // successful captures only (not errors)
+      screensErrored,
+      // pages visited but failed to capture
+      screensSkippedToNextSegment: pendingUrls.length,
+      isPartial: hasMoreSegments,
+      // true = more segments auto-queued
+      discovered,
+      apiEndpoints: capturedApiEndpoints,
+      performanceMetrics,
+      pendingUrls,
+      // feed into next segment's resumeUrls
+      segmentIndex: segIdx,
+      // which segment this was
+      hasMoreSegments,
+      domHashes: Array.from(seenDomHashes)
+      // carry forward to next segment to avoid re-screenshotting
+    };
+  } finally {
+    if (stagehandPromptTokens > 0 || stagehandCompletionTokens > 0) {
+      const totalTokens = stagehandPromptTokens + stagehandCompletionTokens;
+      const shProvider = stagehandModel.includes("/") ? stagehandModel.split("/")[0] : configuredProvider || "openai";
+      const shModel = stagehandModel.includes("/") ? stagehandModel.split("/").slice(1).join("/") : stagehandModel;
+      const shUsage = {
+        provider: shProvider,
+        model: shModel,
+        promptTokens: stagehandPromptTokens,
+        completionTokens: stagehandCompletionTokens,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        totalTokens
+      };
+      recordLLMCall({
+        tenantId: opts.tenantId,
+        projectId: opts.projectId,
+        jobId: opts.jobId,
+        agentType: "crawl_stagehand",
+        label: "stagehand_browser_actions",
+        usage: { ...shUsage, estimatedCostUsd: estimateCost(shUsage, `${shProvider}/${shModel}`) }
+      }).catch(() => {
+      });
+    }
+    await playwrightBrowser?.close().catch(() => {
+    });
+    await stagehand.close().catch(() => {
+    });
+    await chromeProc.close().catch(() => {
+    });
+  }
+}
+const ELEMENT_NOISE = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
+async function extractElements(stagehand, page, llmCfg) {
+  const domElements = await domFallbackElements(page);
+  if (domElements.length >= 2) return domElements;
+  const contentElements = await contentFallbackElements(page);
+  if (contentElements.length > 0) return [...domElements, ...contentElements];
+  try {
+    const extracted = await Promise.race([
+      stagehand.extract(
+        'Extract ONLY the meaningful form and action elements on this page \u2014 inputs, selects, textareas, and action buttons. For each: a clear human-readable purpose (e.g. "Email address input", "Submit password reset button"), its role (button/input/select/checkbox/textarea), and for inputs the expected data type (valid_email/password/phone/date/search_term/text). SKIP: navigation links, header/footer links, cookie banners, theme toggles (dark mode/light mode), social share buttons, version strings, decorative elements, any button inside a nav or header. Focus ONLY on main content forms and primary action buttons. Return at most 20 elements.',
+        ElementSchema
+      ),
+      new Promise((resolve) => setTimeout(() => resolve(null), 3e4))
+    ]);
+    if (!extracted) return domElements;
+    const filtered = extracted.elements.filter((e) => !ELEMENT_NOISE.test(e.meaning ?? ""));
+    return filtered.slice(0, 20);
+  } catch {
+    return domElements;
+  }
+}
+async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false) {
+  const navExpanded = await expandNavMenus(page);
+  const domLinks = await domFallbackLinks(page, origin);
+  const interactiveLinks = await interactAndCollectLinks(page, origin);
+  if (navExpanded) {
+    page.keyboard?.press("Escape").catch(() => {
+    });
+    await page.evaluate(() => {
+      document.querySelectorAll('[aria-expanded="true"],[data-state="open"],[data-headlessui-state="open"]').forEach((el) => {
+        try {
+          el.click();
+        } catch {
+        }
+      });
+    }).catch(() => {
+    });
+  }
+  const allDomLinks = Array.from(/* @__PURE__ */ new Set([...domLinks, ...interactiveLinks]));
+  if (allDomLinks.length >= 30 || skipAi) return allDomLinks;
+  try {
+    const extracted = await Promise.race([
+      stagehand.extract(
+        `Find all in-app navigation links for this application (origin: ${origin}). Include: main nav, sidebar, content links, feature pages, settings pages. Exclude: logout, sign out, delete, external sites, anchor-only links (#), mailto:, javascript:.`,
+        LinksSchema
+      ),
+      new Promise((resolve) => setTimeout(() => resolve(null), 12e3))
+    ]);
+    if (!extracted) return allDomLinks;
+    const { links } = extracted;
+    const currentUrl = page.url?.() ?? "";
+    const aiLinks = links.map((href) => {
+      try {
+        const u = new URL(href, currentUrl);
+        if (!/^#!?\//.test(u.hash)) u.hash = "";
+        return u.toString();
+      } catch {
+        return null;
+      }
+    }).filter((u) => {
+      if (!u) return false;
+      return isUsefulNavigationUrl(u, origin);
+    });
+    return Array.from(/* @__PURE__ */ new Set([...allDomLinks, ...aiLinks])).filter((href) => isUsefulNavigationUrl(href, origin));
+  } catch (err) {
+    const is429 = err?.status === 429 || String(err?.message ?? "").includes("429") || String(err?.message ?? "").includes("rate limit");
+    if (is429) throw Object.assign(new Error("RATE_LIMITED"), { isRateLimit: true, domLinks: allDomLinks });
+    return allDomLinks;
+  }
+}
+async function domFallbackElements(page) {
+  return page.evaluate(() => {
+    const out = [];
+    const seen = /* @__PURE__ */ new Set();
+    const noise = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
+    const querySelectorAllDeep = (root, selector2) => {
+      const results = Array.from(root.querySelectorAll(selector2));
+      root.querySelectorAll("*").forEach((el) => {
+        if (el.shadowRoot) {
+          results.push(...querySelectorAllDeep(el.shadowRoot, selector2));
+        }
+      });
+      return results;
+    };
+    const selector = 'input:not([type=hidden]),select,textarea,button,[role="button"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],a[role="button"],[contenteditable="true"]';
+    querySelectorAllDeep(document, selector).forEach((el) => {
+      const style = window.getComputedStyle(el);
+      if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0" || style.pointerEvents === "none") return;
+      if (el.hasAttribute("inert") || el.getAttribute("aria-disabled") === "true") return;
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 && rect.height === 0) return;
+      const tag = el.tagName.toLowerCase();
+      const ariaRole = el.getAttribute("role");
+      const isActionEl = tag === "button" || tag === "a" || ariaRole === "button";
+      if (tag === "a" || ariaRole === "button" && tag !== "button") {
+        let p = el.parentElement;
+        while (p) {
+          const ptag = p.tagName?.toLowerCase();
+          if (["header", "footer"].includes(ptag)) return;
+          p = p.parentElement;
+        }
+      }
+      let labelText = "";
+      if (el.id) {
+        const linkedLabel = document.querySelector(`label[for="${el.id}"]`);
+        if (linkedLabel) labelText = linkedLabel.textContent?.trim() || "";
+      }
+      if (!labelText) {
+        const wrappingLabel = el.closest("label");
+        if (wrappingLabel) {
+          const clone = wrappingLabel.cloneNode(true);
+          clone.querySelectorAll("input,select,textarea,button").forEach((c) => c.remove());
+          labelText = clone.textContent?.replace(/\s+/g, " ").trim() || "";
+        }
+      }
+      const label = (el.getAttribute("aria-label") || el.getAttribute("placeholder") || labelText || el.innerText?.replace(/\s+/g, " ").trim() || el.getAttribute("name") || "").slice(0, 80);
+      if (!label || noise.test(label) || label.length > 70) return;
+      const role = tag === "input" ? el.type || "text" : el.hasAttribute("contenteditable") ? "rich-text" : ariaRole || tag;
+      const key = `${role}:${label.toLowerCase()}`;
+      if (seen.has(key)) return;
+      seen.add(key);
+      let expectedData;
+      if (tag === "input") {
+        if (el.type === "email" || /email/i.test(label)) expectedData = "valid_email";
+        else if (el.type === "password" || /password/i.test(label)) expectedData = "password";
+        else if (el.type === "tel" || /phone|mobile/i.test(label)) expectedData = "phone";
+        else if (el.type === "date" || /\bdate\b/i.test(label)) expectedData = "date";
+        else if (/\b(code|otp|token)\b/i.test(label)) expectedData = "one_time_code";
+        else if (/\b(search|query)\b/i.test(label)) expectedData = "search_term";
+        else if (el.type === "number" || /\b(amount|quantity|count|age|year)\b/i.test(label)) expectedData = "number";
+        else if (el.type === "url" || /\b(url|website|link)\b/i.test(label)) expectedData = "url";
+        else if (el.type === "file") expectedData = "file";
+      }
+      const attrs = {};
+      if (el.id) attrs.id = el.id;
+      if (el.name) attrs.name = el.name;
+      if (el.getAttribute("data-testid")) attrs["data-testid"] = el.getAttribute("data-testid");
+      if (tag === "input" || tag === "textarea") {
+        if (el.required) attrs.required = "true";
+        if (el.pattern) attrs.pattern = el.pattern;
+        if (el.minLength > 0) attrs.minLength = String(el.minLength);
+        if (el.maxLength > 0 && el.maxLength < 524288) attrs.maxLength = String(el.maxLength);
+        if (el.min !== "") attrs.min = String(el.min);
+        if (el.max !== "") attrs.max = String(el.max);
+        if (el.step && el.step !== "any") attrs.step = String(el.step);
+        if (el.accept) attrs.accept = el.accept;
+        if (el.multiple) attrs.multiple = "true";
+        if (el.readOnly) attrs.readOnly = "true";
+        if (el.disabled) attrs.disabled = "true";
+        if (el.autocomplete && el.autocomplete !== "on") attrs.autocomplete = el.autocomplete;
+      }
+      const cssClass = (el.className || "").split(" ").map((c) => c.trim()).find((c) => c.length > 2 && !/^(true|false|active|disabled|hidden|visible)$/i.test(c));
+      if (cssClass) attrs.cssClass = cssClass;
+      if (tag === "select" && el.options?.length > 0 && el.options.length <= 60) {
+        const opts = Array.from(el.options).map((o) => {
+          const entry = { v: o.value, l: o.text.trim() };
+          if (o.selected) entry.s = true;
+          return entry;
+        }).filter((o) => o.l);
+        if (opts.length > 0) attrs.selectOptions = JSON.stringify(opts);
+      }
+      const describedById = el.getAttribute("aria-describedby");
+      if (describedById) {
+        const helpEl = document.getElementById(describedById.split(" ")[0]);
+        const helpText = helpEl?.textContent?.trim().slice(0, 200);
+        if (helpText) attrs.helpText = helpText;
+      }
+      const form = el.closest("form");
+      if (form) {
+        if (form.id) attrs.formId = form.id;
+        const formAction = form.getAttribute("action");
+        if (formAction && !formAction.startsWith("javascript:")) attrs.formAction = formAction.slice(0, 120);
+        const formMethod = form.method;
+        if (formMethod) attrs.formMethod = formMethod.toUpperCase();
+      }
+      const fieldset = el.closest("fieldset");
+      if (fieldset) {
+        const legendText = fieldset.querySelector("legend")?.textContent?.trim().slice(0, 60);
+        if (legendText) attrs.fieldsetGroup = legendText;
+      }
+      if (tag === "button" || ariaRole === "button") {
+        if (el.type && el.type !== "button") attrs.buttonType = el.type;
+        if (/\b(delete|remove|destroy|revoke|clear|reset|discard|cancel)\b/i.test(label)) attrs.isDestructive = "true";
+      }
+      const stateMap = {};
+      const ariaExpanded = el.getAttribute("aria-expanded");
+      const ariaChecked = el.getAttribute("aria-checked");
+      const ariaPressed = el.getAttribute("aria-pressed");
+      const ariaSelected = el.getAttribute("aria-selected");
+      const ariaInvalid = el.getAttribute("aria-invalid");
+      const ariaCurrent = el.getAttribute("aria-current");
+      if (ariaExpanded !== null) stateMap.expanded = ariaExpanded;
+      if (ariaChecked !== null) stateMap.checked = ariaChecked;
+      if (ariaPressed !== null) stateMap.pressed = ariaPressed;
+      if (ariaSelected !== null) stateMap.selected = ariaSelected;
+      if (ariaInvalid && ariaInvalid !== "false") stateMap.invalid = ariaInvalid;
+      if (ariaCurrent) stateMap.current = ariaCurrent;
+      if (el.getAttribute("aria-required") === "true" && !attrs.required) stateMap.required = "true";
+      const getRegion = (node) => {
+        let p = node.parentElement;
+        while (p && p !== document.body) {
+          const ptag = p.tagName?.toLowerCase();
+          const prole = p.getAttribute?.("role");
+          if (prole === "dialog" || prole === "alertdialog") return "modal";
+          if (prole === "main" || ptag === "main") return "main";
+          if (prole === "complementary" || ptag === "aside") return "aside";
+          if (prole === "navigation" || ptag === "nav") return "nav";
+          if (prole === "banner" || ptag === "header") return "header";
+          if (prole === "contentinfo" || ptag === "footer") return "footer";
+          p = p.parentElement;
+        }
+        return void 0;
+      };
+      const parentLandmark = getRegion(el);
+      if (el.hasAttribute("contenteditable")) {
+        const cls = ((el.className || "") + " " + (el.getAttribute("data-type") || "")).toLowerCase();
+        const editorType = cls.includes("tiptap") || cls.includes("prosemirror") ? "tiptap" : cls.includes("cm-") || cls.includes("codemirror") ? "codemirror" : window.Quill ? "quill" : typeof window.monaco !== "undefined" ? "monaco" : "contenteditable";
+        attrs.editorType = editorType;
+      }
+      const notes = Object.keys(attrs).length > 0 ? JSON.stringify(attrs) : void 0;
+      const boundingRect = {
+        x: Math.round(rect.x),
+        y: Math.round(rect.y),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        top: Math.round(rect.top),
+        left: Math.round(rect.left),
+        right: Math.round(rect.right),
+        bottom: Math.round(rect.bottom)
+      };
+      out.push({
+        meaning: label,
+        role,
+        expectedData,
+        notes,
+        boundingRect,
+        ariaState: Object.keys(stateMap).length > 0 ? stateMap : void 0,
+        parentLandmark: parentLandmark ?? void 0
+      });
+    });
+    const priority = (e) => ["email", "password", "text", "tel", "date", "number", "select", "textarea", "url", "file"].includes(e.role) ? 0 : 1;
+    out.sort((a, b) => priority(a) - priority(b));
+    return out;
+  }).catch(() => []);
+}
+async function contentFallbackElements(page) {
+  return page.evaluate(() => {
+    const out = [];
+    const seen = /* @__PURE__ */ new Set();
+    const addUniq = (item) => {
+      const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
+      if (!seen.has(key) && item.meaning.trim().length > 0) {
+        seen.add(key);
+        out.push(item);
+      }
+    };
+    const title = document.title?.trim();
+    if (title) addUniq({ meaning: title, role: "page-title", notes: JSON.stringify({ tag: "title" }) });
+    const desc = document.querySelector('meta[name="description"]')?.getAttribute("content")?.trim();
+    if (desc) addUniq({ meaning: desc, role: "meta-description", notes: JSON.stringify({ tag: "meta[name=description]" }) });
+    const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim();
+    if (ogTitle) addUniq({ meaning: ogTitle, role: "og-title", notes: JSON.stringify({ tag: "meta[property=og:title]" }) });
+    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim();
+    if (canonical) addUniq({ meaning: canonical, role: "canonical-url", notes: JSON.stringify({ tag: "link[rel=canonical]" }) });
+    document.querySelectorAll("h1, h2, h3").forEach((h) => {
+      const text = h.textContent?.replace(/\s+/g, " ").trim().slice(0, 80);
+      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
+    });
+    document.querySelectorAll("img[alt]").forEach((img) => {
+      const alt = img.getAttribute("alt")?.trim().slice(0, 80);
+      const src = img.getAttribute("src")?.slice(0, 120);
+      if (alt) addUniq({ meaning: alt, role: "image", notes: JSON.stringify({ src }) });
+    });
+    const navTags = /* @__PURE__ */ new Set(["nav", "header", "footer"]);
+    document.querySelectorAll("a[href]").forEach((a) => {
+      let p = a.parentElement;
+      while (p) {
+        if (navTags.has(p.tagName?.toLowerCase())) return;
+        p = p.parentElement;
+      }
+      const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 80);
+      const href = a.href;
+      if (label && href && !/^(#|javascript:|mailto:)/i.test(href)) {
+        const isExternal = !href.includes(location.hostname);
+        addUniq({ meaning: label, role: isExternal ? "external-link" : "internal-link", notes: JSON.stringify({ href: href.slice(0, 120) }) });
+      }
+    });
+    const hasAnimation = !!document.querySelector('[class*="animate"], [class*="motion"], [class*="transition"], [class*="scroll-"], [data-aos], [data-scroll]');
+    if (hasAnimation) addUniq({ meaning: "CSS animations / scroll effects detected on page", role: "animation-layer", notes: JSON.stringify({ detected: true }) });
+    if (document.querySelector("video")) addUniq({ meaning: "Video element present", role: "video", notes: JSON.stringify({ count: document.querySelectorAll("video").length }) });
+    if (document.querySelector("canvas")) addUniq({ meaning: "Canvas element present", role: "canvas" });
+    return out.slice(0, 30);
+  }).catch(() => []);
+}
+async function interactAndCollectLinks(page, origin) {
+  const collected = /* @__PURE__ */ new Set();
+  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+  const collectCurrentLinks = async () => {
+    const links = await page.evaluate((origin2) => {
+      const out = [];
+      const logout = /logout|log out|sign out|delete/i;
+      document.querySelectorAll("a[href]").forEach((a) => {
+        if (logout.test(a.textContent ?? "")) return;
+        try {
+          const u = new URL(a.href);
+          if (u.origin === origin2) {
+            if (u.hash && !/^#!?\//.test(u.hash)) u.hash = "";
+            out.push(u.toString());
+          }
+        } catch {
+        }
+      });
+      return out;
+    }, origin).catch(() => []);
+    links.forEach((l) => collected.add(l));
+  };
+  try {
+    const tabCount = await page.evaluate(
+      () => document.querySelectorAll('[role="tab"]').length
+    ).catch(() => 0);
+    for (let i = 0; i < Math.min(tabCount, 15); i++) {
+      try {
+        const clicked = await page.evaluate((idx) => {
+          const tabs = document.querySelectorAll('[role="tab"]');
+          const tab = tabs[idx];
+          if (!tab) return false;
+          const s = window.getComputedStyle(tab);
+          const r = tab.getBoundingClientRect();
+          if (s.display === "none" || r.width < 4 || r.height < 4) return false;
+          tab.click();
+          return true;
+        }, i).catch(() => false);
+        if (clicked) {
+          await sleep(350);
+          await collectCurrentLinks();
+        }
+      } catch {
+      }
+    }
+    const carouselNextSel = [
+      'button[aria-label*="next" i]',
+      'button[aria-label*="forward" i]',
+      'button[aria-label*="right" i]',
+      '[data-direction="next"]',
+      '[class*="carousel" i] button[class*="next" i]',
+      '[class*="slider" i] button[class*="next" i]'
+    ].join(", ");
+    for (let i = 0; i < 8; i++) {
+      const advanced = await page.evaluate((sel) => {
+        const btn = document.querySelector(sel);
+        if (!btn) return false;
+        const s = window.getComputedStyle(btn);
+        const r = btn.getBoundingClientRect();
+        if (s.display === "none" || r.width < 4 || r.height < 4) return false;
+        btn.click();
+        return true;
+      }, carouselNextSel).catch(() => false);
+      if (!advanced) break;
+      await sleep(300);
+      await collectCurrentLinks();
+    }
+    const accordionCount = await page.evaluate(
+      () => document.querySelectorAll('details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]').length
+    ).catch(() => 0);
+    for (let i = 0; i < Math.min(accordionCount, 10); i++) {
+      try {
+        await page.evaluate((idx) => {
+          const items = document.querySelectorAll(
+            'details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]'
+          );
+          const el = items[idx];
+          if (el) {
+            try {
+              el.click();
+            } catch {
+            }
+          }
+        }, i).catch(() => {
+        });
+        await sleep(250);
+        await collectCurrentLinks();
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return Array.from(collected);
+}
+async function expandNavMenus(page) {
+  try {
+    const clicked = await page.evaluate(() => {
+      const isVisible = (el) => {
+        const s = window.getComputedStyle(el);
+        if (s.display === "none" || s.visibility === "hidden" || parseFloat(s.opacity) < 0.1) return false;
+        const r = el.getBoundingClientRect();
+        return r.width > 4 && r.height > 4;
+      };
+      const inBadContext = (el) => {
+        let p = el.parentElement;
+        while (p) {
+          const tag = p.tagName?.toLowerCase();
+          const role = p.getAttribute("role") ?? "";
+          if (["form", "table", "select"].includes(tag ?? "") || ["dialog", "alertdialog", "combobox", "listbox", "grid", "treegrid"].includes(role)) return true;
+          p = p.parentElement;
+        }
+        return false;
+      };
+      const targets = /* @__PURE__ */ new Set();
+      document.querySelectorAll('[aria-haspopup]:not([aria-expanded="true"])').forEach((el) => {
+        if (isVisible(el) && !inBadContext(el)) targets.add(el);
+      });
+      document.querySelectorAll('[aria-expanded="false"]').forEach((el) => {
+        if ((el.tagName === "BUTTON" || el.getAttribute("role") === "button") && isVisible(el) && !inBadContext(el)) targets.add(el);
+      });
+      document.querySelectorAll('[data-state="closed"],[data-headlessui-state="closed"]').forEach((el) => {
+        if ((el.tagName === "BUTTON" || el.getAttribute("role") === "button") && isVisible(el) && !inBadContext(el)) targets.add(el);
+      });
+      let count = 0;
+      targets.forEach((el) => {
+        try {
+          el.click();
+          count++;
+        } catch {
+        }
+      });
+      return count;
+    });
+    if (clicked > 0) await new Promise((r) => setTimeout(r, 500));
+    try {
+      const navHandles = await page.$$(
+        'nav > ul > li, nav > li, header > ul > li, header > nav > ul > li, [role="navigation"] > ul > li, [role="menubar"] > [role="menuitem"], [class*="nav"] > ul > li, [class*="menu"] > li'
+      ).catch(() => []);
+      for (const h of navHandles.slice(0, 20)) {
+        try {
+          await h.hover({ force: true, timeout: 600 });
+          await page.waitForTimeout(280);
+        } catch {
+        }
+      }
+      if (navHandles.length > 0) {
+        await page.mouse.move(2, 2);
+        await page.waitForTimeout(150);
+      }
+    } catch {
+    }
+    return clicked > 0;
+  } catch {
+    return false;
+  }
+}
+async function domFallbackLinks(page, origin) {
+  return page.evaluate((origin2) => {
+    const logout = /logout|log out|sign out|delete/i;
+    const isRouteHash = (hash) => /^#!?\//.test(hash);
+    const seen = /* @__PURE__ */ new Set();
+    const add = (rawHref, textContent) => {
+      if (logout.test(textContent)) return;
+      if (!rawHref) return;
+      if (rawHref.startsWith("#") && !isRouteHash(rawHref)) return;
+      if (rawHref.startsWith("mailto:") || rawHref.startsWith("javascript:")) return;
+      try {
+        const u = new URL(rawHref, location.href);
+        if (!isRouteHash(u.hash)) u.hash = "";
+        if (new URL(u).origin === origin2) seen.add(u.toString());
+      } catch {
+      }
+    };
+    const allRoots = [document];
+    const rootSeen = /* @__PURE__ */ new WeakSet();
+    rootSeen.add(document);
+    while (allRoots.length > 0) {
+      const root = allRoots.shift();
+      if (!root) continue;
+      try {
+        root.querySelectorAll("a[href]").forEach((a) => add(a.href || a.getAttribute("href") || "", a.textContent || ""));
+        root.querySelectorAll("[href]:not(a):not(link):not(base):not(script)").forEach((el) => {
+          add(el.getAttribute("href") || "", el.textContent || "");
+        });
+        root.querySelectorAll("[data-href]").forEach((el) => {
+          add(el.getAttribute("data-href") || "", el.textContent || "");
+        });
+        root.querySelectorAll("*").forEach((el) => {
+          try {
+            if (el.shadowRoot && !rootSeen.has(el.shadowRoot)) {
+              rootSeen.add(el.shadowRoot);
+              allRoots.push(el.shadowRoot);
+            }
+          } catch (_) {
+          }
+        });
+      } catch (_) {
+      }
+    }
+    return Array.from(seen);
+  }, origin).catch(() => []);
+}
+async function extractLinkMappings(page, origin) {
+  return page.evaluate((origin2) => {
+    const logout = /logout|log out|sign out|delete/i;
+    return Array.from(document.querySelectorAll("a[href]")).filter((a) => !logout.test(a.textContent ?? "")).map((a) => {
+      const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || a.getAttribute("title") || "").slice(0, 80);
+      const href = (() => {
+        try {
+          const u = new URL(a.href);
+          if (!/^#!?\//.test(u.hash)) u.hash = "";
+          return u.toString();
+        } catch {
+          return "";
+        }
+      })();
+      return { label, href };
+    }).filter(
+      ({ label, href }) => label.length > 2 && href.length > 0 && (() => {
+        try {
+          return new URL(href).origin === origin2;
+        } catch {
+          return false;
+        }
+      })()
+    );
+  }, origin).catch(() => []);
+}
+async function computeDomHash(page) {
+  return page.evaluate(() => {
+    let hashStr = "";
+    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
+    let node;
+    while (node = walker.nextNode()) {
+      const el = node;
+      if (/^(MAIN|SECTION|NAV|FORM|INPUT|SELECT|TEXTAREA|BUTTON|TABLE|TR|TD|TH|UL|LI|A|H1|H2|H3)$/.test(el.tagName)) {
+        let depth = 0;
+        let parent = el.parentElement;
+        while (parent) {
+          if (/^(MAIN|SECTION|NAV|FORM|INPUT|SELECT|TEXTAREA|BUTTON|TABLE|TR|TD|TH|UL|LI|A|H1|H2|H3)$/.test(parent.tagName)) depth++;
+          parent = parent.parentElement;
+        }
+        hashStr += el.tagName + depth + "|";
+      }
+    }
+    let hash = 5381;
+    for (let i = 0; i < hashStr.length; i++) {
+      hash = hash * 33 ^ hashStr.charCodeAt(i);
+    }
+    return (hash >>> 0).toString(16);
+  }).catch(() => "");
+}
+async function waitForURLChange(page, fromUrl, timeout = 1e4) {
+  const deadline = Date.now() + timeout;
+  while (Date.now() < deadline) {
+    const current = page.url();
+    try {
+      const from = new URL(fromUrl);
+      const now = new URL(current);
+      if (now.pathname !== from.pathname || now.origin !== from.origin) return;
+    } catch {
+      if (current !== fromUrl) return;
+    }
+    await new Promise((r) => setTimeout(r, 200));
+  }
+}
+async function dismissOverlays(page) {
+  const selectors = [
+    // OneTrust / CookieBot / TrustArc (most common GDPR banners)
+    "#onetrust-accept-btn-handler",
+    "#accept-cookie-consent",
+    ".cc-btn.cc-allow",
+    "[data-cookiebanner] button",
+    '[id*="cookie-accept"]',
+    '[class*="cookie-accept"]',
+    '[id*="cookieAccept"]',
+    '[class*="cookieAccept"]',
+    'button[aria-label*="Accept cookies" i]',
+    'button[aria-label*="Accept all" i]',
+    'button[title*="Accept" i][class*="cookie" i]',
+    // Intercom / Drift / HubSpot chat
+    '[class*="intercom-close"]',
+    '#hubspot-messages-iframe-container [aria-label*="close" i]',
+    // Generic overlay close buttons
+    '[role="dialog"] button[aria-label="Close" i]',
+    '[role="dialog"] button[aria-label="Dismiss" i]',
+    // Newsletter popups
+    'button[class*="modal-close"]',
+    'button[class*="popup-close"]',
+    '[class*="newsletter"] button[class*="close"]'
+  ];
+  for (const sel of selectors) {
+    try {
+      const el = page.locator(sel).first();
+      if (await el.isVisible({ timeout: 300 }).catch(() => false)) {
+        await el.click({ timeout: 500, force: true }).catch(() => {
+        });
+      }
+    } catch {
+    }
+  }
+  await page.waitForTimeout(300).catch(() => {
+  });
+}
+async function scrollToTriggerLazyLoad(page) {
+  try {
+    await page.evaluate(async () => {
+      const step = Math.max(window.innerHeight, 400);
+      const total = Math.min(document.body.scrollHeight, 12e3);
+      for (let y = step; y < total; y += step) {
+        window.scrollTo(0, y);
+        await new Promise((r) => setTimeout(r, 120));
+      }
+      window.scrollTo(0, 0);
+    });
+    await page.waitForTimeout(400).catch(() => {
+    });
+    await waitForStableDOM(page, 3e3, 300);
+  } catch {
+  }
+}
+async function waitForStableDOM(page, maxWait = 8e3, stabilityMs = 400) {
+  const deadline = Date.now() + maxWait;
+  let lastCount = -1;
+  let stableAt = 0;
+  while (Date.now() < deadline) {
+    const count = await page.evaluate(() => document.querySelectorAll("*").length).catch(() => 0);
+    if (count !== lastCount) {
+      lastCount = count;
+      stableAt = Date.now();
+    } else if (Date.now() - stableAt >= stabilityMs) {
+      return;
+    }
+    await new Promise((r) => setTimeout(r, 100));
+  }
+}
+async function waitForScreenshotReady(page) {
+  try {
+    await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => {
+    });
+    await page.waitForLoadState("networkidle", { timeout: 12e3 }).catch(() => {
+    });
+    await page.evaluate(() => {
+      if (document.readyState === "complete") return true;
+      return new Promise((resolve) => {
+        const done = () => resolve(true);
+        window.addEventListener("load", done, { once: true });
+        setTimeout(done, 4e3);
+      });
+    }).catch(() => {
+    });
+    await page.evaluate(async () => {
+      const fonts = document.fonts;
+      if (fonts?.ready) await fonts.ready;
+    }).catch(() => {
+    });
+    await scrollToTriggerLazyLoad(page);
+    await waitForStableDOM(page, 1e4, 700);
+    await page.evaluate(async () => {
+      const pending = Array.from(document.images).filter((img) => !img.complete);
+      if (pending.length === 0) return;
+      await Promise.race([
+        Promise.all(pending.map((img) => new Promise((resolve) => {
+          img.addEventListener("load", () => resolve(), { once: true });
+          img.addEventListener("error", () => resolve(), { once: true });
+        }))),
+        new Promise((resolve) => setTimeout(resolve, 4e3))
+      ]);
+    }).catch(() => {
+    });
+    await waitForStableDOM(page, 4e3, 500);
+    await page.waitForTimeout(250).catch(() => {
+    });
+  } catch {
+  }
+}
+async function hasMeaningfulAuthOrPublicState(page, requestedUrl) {
+  const pathLooksRelevant = (() => {
+    try {
+      const pathname = new URL(page.url?.() ?? requestedUrl).pathname;
+      return AUTH_PATH_RE.test(pathname) || PUBLIC_PATH_RE.test(pathname);
+    } catch {
+      return false;
+    }
+  })();
+  const domLooksRelevant = await page.evaluate(() => {
+    const text = (document.body?.innerText ?? "").toLowerCase();
+    const inputs = Array.from(document.querySelectorAll("input"));
+    const hasPassword = inputs.some((i) => i.type === "password");
+    const hasIdentityInput = inputs.some((i) => {
+      const meta = `${i.type} ${i.name} ${i.id} ${i.placeholder} ${i.autocomplete}`.toLowerCase();
+      return /(email|username|user|login|phone|otp|code|password)/.test(meta);
+    });
+    const hasAuthCopy = /(sign in|log in|login|sign up|create account|register|forgot password|reset password|verify email|one-time|otp|continue with email)/i.test(text);
+    const hasPublicNav = /(pricing|contact|features|documentation|docs|about|privacy|terms)/i.test(text);
+    return hasPassword || hasIdentityInput || hasAuthCopy || hasPublicNav;
+  }).catch(() => false);
+  return pathLooksRelevant || domLooksRelevant;
+}
+async function discoverAuthCandidates(page, origin) {
+  return page.evaluate((siteOrigin) => {
+    const authTextRe = /(sign\s*in|log\s*in|login|sign\s*up|signup|register|create account|forgot password|reset password|use email|continue with email|back to login|verify|invite|get started|start free trial)/i;
+    const skipTextRe = /(logout|log out|sign out|delete|remove|discard|cancel subscription|payment|billing)/i;
+    const authPathRe = /\/(login|log-in|signin|sign-in|auth|oauth|sso|account\/login|user\/login|app\/login|register|signup|sign-up|create-account|forgot|reset|password|verify-email|invite)(\/|$|\?)/i;
+    const publicPathRe = /\/($|pricing|contact|about|help|docs|features|security|privacy|terms)(\/|$|\?)/i;
+    const sameSiteBrowser = (href) => {
+      try {
+        const u = new URL(href, location.href);
+        const s = new URL(siteOrigin);
+        const strip = (h) => h.replace(/^www\./i, "");
+        return strip(u.hostname) === strip(s.hostname);
+      } catch {
+        return false;
+      }
+    };
+    const flowTypeFor = (value) => {
+      const v = value.toLowerCase();
+      if (/(forgot|reset|password)/.test(v)) return "password_reset";
+      if (/(register|signup|sign up|create account|join)/.test(v)) return "register";
+      if (/(verify|verification|otp|mfa|2fa|invite)/.test(v)) return v.includes("invite") ? "invite" : "verification";
+      if (/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)/.test(v)) return "session_recovery";
+      if (/(login|log in|signin|sign in|auth|sso|oauth)/.test(v)) return "login";
+      return "public";
+    };
+    const textOf = (el) => (el.textContent || el.getAttribute("aria-label") || el.getAttribute("title") || el.getAttribute("placeholder") || "").replace(/\s+/g, " ").trim();
+    const isVisible = (el) => {
+      const rect = el.getBoundingClientRect?.();
+      const style = window.getComputedStyle(el);
+      return !!rect && rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
+    };
+    const links = /* @__PURE__ */ new Map();
+    const addHref = (hrefRaw, labelRaw) => {
+      if (!hrefRaw || /^(javascript|mailto|tel):/i.test(hrefRaw)) return;
+      let href = "";
+      try {
+        href = new URL(hrefRaw, location.href).toString();
+      } catch {
+        return;
+      }
+      if (!sameSiteBrowser(href)) return;
+      const u = new URL(href);
+      const label = labelRaw || u.pathname;
+      if (skipTextRe.test(label) || skipTextRe.test(u.pathname)) return;
+      if (!authTextRe.test(label) && !authPathRe.test(u.pathname) && !publicPathRe.test(u.pathname)) return;
+      const key = `${u.origin}${u.pathname.replace(/\/$/, "") || "/"}${u.search}`;
+      if (!links.has(key)) links.set(key, { href, label, flowType: flowTypeFor(`${label} ${u.pathname}`) });
+    };
+    const walk = (root) => {
+      root.querySelectorAll("a[href]").forEach((el) => addHref(el.href || el.getAttribute("href"), textOf(el)));
+      root.querySelectorAll("[data-href],[data-url],[data-link]").forEach((el) => addHref(el.dataset?.href || el.dataset?.url || el.dataset?.link || "", textOf(el)));
+      root.querySelectorAll("*").forEach((el) => {
+        if (el.shadowRoot) walk(el.shadowRoot);
+      });
+    };
+    walk(document);
+    const clicks = [];
+    const clickEls = Array.from(document.querySelectorAll('button,[role="button"],a:not([href]),[data-state],[aria-haspopup]'));
+    let idx = 0;
+    for (const el of clickEls) {
+      if (clicks.length >= 12) break;
+      if (!isVisible(el)) continue;
+      const label = textOf(el);
+      if (!label || !authTextRe.test(label) || skipTextRe.test(label)) continue;
+      const type = (el.getAttribute("type") || "").toLowerCase();
+      const inCredentialForm = !!el.closest("form")?.querySelector('input[type="password"],input[type="email"],input[name*="user" i],input[name*="email" i]');
+      const submitLike = type === "submit" || /^(login|log in|sign in|continue)$/i.test(label);
+      if (inCredentialForm && submitLike && !/(forgot|reset|register|sign up|create account|back to login|use email)/i.test(label)) continue;
+      const attr = `zeta-auth-${idx++}`;
+      el.setAttribute("data-zeta-auth-discovery-id", attr);
+      clicks.push({ selector: `[data-zeta-auth-discovery-id="${attr}"]`, label, flowType: flowTypeFor(label) });
+    }
+    return { links: Array.from(links.values()).slice(0, 32), clicks };
+  }, origin).catch(() => ({ links: [], clicks: [] }));
+}
+async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordinal, trigger) {
+  const finalUrl = page.url?.() ?? requestedUrl;
+  if (finalUrl.startsWith("chrome-error://")) return { captured: false, finalUrl, flowType: "public" };
+  if (!await hasMeaningfulAuthOrPublicState(page, requestedUrl)) {
+    return { captured: false, finalUrl, flowType: flowTypeForUrlOrText(`${requestedUrl} ${finalUrl} ${trigger?.label ?? ""}`) };
+  }
+  const flowType = trigger?.flowType ?? flowTypeForUrlOrText(`${requestedUrl} ${finalUrl} ${trigger?.label ?? ""}`);
+  const title = await page.title().catch(() => "");
+  const baseName = title?.trim() && !/^(untitled|about:blank)$/i.test(title.trim()) ? title.trim().slice(0, 72) : titleFromUrl(finalUrl);
+  const prefix = flowType === "public" ? "Public" : `Auth ${flowType.replace("_", " ")}`;
+  const name = `${prefix} \xB7 ${trigger?.label ? trigger.label.slice(0, 48) : baseName}`.replace(/\s+/g, " ");
+  const screenUrl = trigger?.sameUrlState ? `${finalUrl.split("#")[0]}#zeta-auth-flow=${compactSlug(trigger.label ?? `${flowType}-${ordinal}`)}` : finalUrl;
+  const shotPath = path.join(opts.screenshotDir ?? "/tmp", `auth-discovery-${opts.jobId ?? "run"}-${ordinal}.png`);
+  await waitForScreenshotReady(page);
+  await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {
+  });
+  const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === "entry" ? "ENTRY_POINT" : "SEED_URL").catch(() => null);
+  if (!screen) return { captured: false, finalUrl, flowType };
+  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP").catch(() => {
+  });
+  await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+    screenId: screen.id,
+    jobId: opts.jobId,
+    type: "AUTH_DISCOVERY",
+    payload: {
+      requestedUrl,
+      finalUrl,
+      screenUrl,
+      authState: flowType === "public" ? "logged_out" : "auth_flow",
+      flowType,
+      source,
+      triggerLabel: trigger?.label ?? null
+    }
+  }).catch(() => {
+  });
+  logger.info({ requestedUrl, finalUrl, screenUrl, flowType, source }, "[crawler] auth-discovery: captured screen");
+  return { captured: true, finalUrl, flowType };
+}
+async function runAuthDiscoveryPhase(page, opts, origin, onProgress) {
+  const queue = [];
+  const queued = /* @__PURE__ */ new Set();
+  const capturedKeys = /* @__PURE__ */ new Set();
+  const discoveredUrls = /* @__PURE__ */ new Set();
+  const enqueue = (url, source) => {
+    if (!shouldSeedAuthDiscoveryUrl(url, origin)) return;
+    const key = normalizeAuthUrl(url);
+    if (queued.has(key) || queue.length >= AUTH_DISCOVERY_MAX_QUEUE) return;
+    queued.add(key);
+    queue.push({ url, source });
+  };
+  enqueue(opts.appUrl, "entry");
+  for (const p of AUTH_DISCOVERY_PATHS) {
+    try {
+      enqueue(new URL(p, origin).toString(), "seed_path");
+    } catch {
+    }
+  }
+  let captured = 0;
+  let ordinal = 0;
+  await onProgress?.(10, "Discovering public and auth flows");
+  while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
+    const current = queue.shift();
+    try {
+      const response = await page.goto(current.url, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => null);
+      if (response && response.status && response.status() >= 400) continue;
+      await page.waitForLoadState("networkidle", { timeout: 6e3 }).catch(() => {
+      });
+      await waitForStableDOM(page);
+      await acceptCookieConsent(page, { waitMs: 400 }).catch(() => {
+      });
+      const finalUrl = page.url?.() ?? current.url;
+      const captureKey = normalizeAuthUrl(finalUrl);
+      if (!capturedKeys.has(captureKey)) {
+        const result = await captureAuthDiscoveryScreen(page, opts, current.url, current.source, ordinal++);
+        if (result.captured) {
+          captured++;
+          capturedKeys.add(captureKey);
+          discoveredUrls.add(result.finalUrl);
+        }
+      }
+      const candidates = await discoverAuthCandidates(page, origin);
+      for (const link of candidates.links) enqueue(link.href, "page_link");
+      const baseUrlForClicks = finalUrl;
+      const clickLimit = Math.min(candidates.clicks.length, AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE);
+      for (let i = 0; i < clickLimit && captured < AUTH_DISCOVERY_MAX_SCREENS; i++) {
+        await page.goto(baseUrlForClicks, { waitUntil: "domcontentloaded", timeout: 12e3 }).catch(() => {
+        });
+        await page.waitForLoadState("networkidle", { timeout: 4e3 }).catch(() => {
+        });
+        await waitForStableDOM(page);
+        const freshCandidates = await discoverAuthCandidates(page, origin);
+        const click = freshCandidates.clicks[i];
+        if (!click) continue;
+        const beforeUrl = page.url?.() ?? baseUrlForClicks;
+        await page.locator(click.selector).first().click({ timeout: 4e3 }).catch(() => {
+        });
+        await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
+        });
+        await waitForStableDOM(page);
+        const afterUrl = page.url?.() ?? beforeUrl;
+        const sameUrlState = normalizeAuthUrl(beforeUrl) === normalizeAuthUrl(afterUrl);
+        const stateKey = sameUrlState ? `${normalizeAuthUrl(afterUrl)}::${compactSlug(click.label)}` : normalizeAuthUrl(afterUrl);
+        if (capturedKeys.has(stateKey)) continue;
+        const result = await captureAuthDiscoveryScreen(page, opts, beforeUrl, "auth_click", ordinal++, {
+          label: click.label,
+          flowType: click.flowType,
+          sameUrlState
+        });
+        if (result.captured) {
+          captured++;
+          capturedKeys.add(stateKey);
+          discoveredUrls.add(result.finalUrl);
+          const nested = await discoverAuthCandidates(page, origin);
+          for (const link of nested.links) enqueue(link.href, "page_link");
+        }
+      }
+    } catch (err) {
+      logger.warn({ url: current.url, err: String(err?.message ?? err) }, "[crawler] auth-discovery: URL failed \u2014 skipping");
+    }
+  }
+  await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => {
+  });
+  await waitForStableDOM(page).catch(() => {
+  });
+  await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+    jobId: opts.jobId,
+    type: "AUTH_DISCOVERY_SUMMARY",
+    payload: { captured, discoveredUrls: Array.from(discoveredUrls), maxScreens: AUTH_DISCOVERY_MAX_SCREENS }
+  }).catch(() => {
+  });
+  logger.info({ captured, discovered: discoveredUrls.size }, "[crawler] auth-discovery: phase complete");
+  return { captured, discoveredUrls: Array.from(discoveredUrls) };
+}
+export {
+  crawlProject,
+  discoverLinksViaBrowser
+};
+//# sourceMappingURL=crawler.js.map