@detiq/crawler 1.1.14 → 1.1.15

package/dist/crawler.js

@@ -50,6 +50,25 @@ import { screensToSitemapUrls, writeSitemap } from "./sitemap-writer.js";
 import { generateOpenApiSpec, specToYaml } from "./openapi-generator.js";
 import { WebhookNotifier } from "./webhook-notifier.js";
 import { generatePlaywrightTest } from "./playwright-recorder.js";
+import { enumerateDropdowns } from "./dropdown-enumerator.js";
+import { probeCrudFlows } from "./crud-interaction.js";
+import { crawlFiltersAndPagination } from "./filter-pagination-crawler.js";
+import { probeAdvancedUI } from "./advanced-ui-probes.js";
+import { ScreenTransitionGraph } from "./state-graph.js";
+import { crawlNestedContexts } from "./nested-window-crawler.js";
+import { crawlShadowDom } from "./shadow-dom-crawler.js";
+import { crawlAccessibilityTree } from "./accessibility-tree-crawler.js";
+import { collectPerfMetrics } from "./performance-metrics.js";
+import { inspectServiceWorker } from "./service-worker-interceptor.js";
+async function getCrawlTuning() {
+  const { getSystemConfig } = await import("@detiq/core");
+  const cfg = await getSystemConfig("crawl_tuning");
+  return {
+    navTimeoutMs: parseInt(String(cfg?.navTimeoutMs ?? "15000"), 10),
+    defaultTimeoutMs: parseInt(String(cfg?.defaultTimeoutMs ?? "2000"), 10),
+    userAgent: String(cfg?.userAgent ?? "ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)")
+  };
+}
 const _crawlLLMCache = /* @__PURE__ */ new Map();
 async function getCachedCrawlLLM(tenantId) {
   if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
@@ -83,20 +102,26 @@ function isGeneratedDynamicPath(pathname) {
   if (ALLOWED_NUMERIC_PATHS.has(normalized)) return false;
   return normalized.split("/").filter(Boolean).some(isGeneratedPathSegment);
 }
+const DOWNLOAD_EXT_RE = /\.(pdf|zip|gz|tar|rar|7z|exe|dmg|pkg|deb|rpm|msi|apk|ipa|doc|docx|xls|xlsx|ppt|pptx|csv|json|xml|yaml|yml|txt|log|sql|db|sqlite|mp4|mp3|avi|mov|mkv|webm|wav|ogg|flac|jpg|jpeg|png|gif|bmp|svg|webp|ico|woff|woff2|ttf|eot|otf|bin|iso|img|vhd|vmdk)(\?.*)?$/i;
+const API_PATH_RE = /^\/api\//i;
+const EXPORT_PATH_RE = /\/(exports?|downloads?|export-csv|export-pdf|export-excel|download-report|generate-report|report-download)(\/|$|\?)/i;
 function isUsefulNavigationUrl(href, siteOrigin) {
   try {
     const u = new URL(href);
     if (!sameSite(u.toString(), siteOrigin)) return false;
     if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
     if (isGeneratedDynamicPath(u.pathname)) return false;
+    if (DOWNLOAD_EXT_RE.test(u.pathname)) return false;
+    if (API_PATH_RE.test(u.pathname)) return false;
+    if (EXPORT_PATH_RE.test(u.pathname)) return false;
     return true;
   } catch {
     return false;
   }
 }
-const AUTH_DISCOVERY_MAX_SCREENS = 36;
-const AUTH_DISCOVERY_MAX_QUEUE = 48;
-const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = 6;
+const AUTH_DISCOVERY_MAX_SCREENS = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_SCREENS ?? "") || 36;
+const AUTH_DISCOVERY_MAX_QUEUE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_QUEUE ?? "") || 48;
+const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE ?? "") || 6;
 const AUTH_DISCOVERY_PATHS = [
   "/",
   "/login",
@@ -322,6 +347,7 @@ const TwoFASchema = z.object({
   fieldLabel: z.string().optional()
 });
 async function crawlProject(opts, onProgress) {
+  const tuning = await getCrawlTuning();
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
   const uploadToCloud = async (localPath, key) => {
     if (!opts.storageConfig) return void 0;
@@ -367,15 +393,15 @@ async function crawlProject(opts, onProgress) {
             break;
           case "openrouter":
             stagehandModel = `openai/${m}`;
-            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, baseURL: baseUrl || "https://openrouter.ai/api/v1" };
+            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, baseURL: baseUrl || process.env.CRAWLER_OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1" };
             break;
           case "ollama":
             stagehandModel = `ollama/${m}`;
-            stagehandModelOpts = { baseURL: baseUrl || "http://localhost:11434" };
+            stagehandModelOpts = { baseURL: baseUrl || process.env.CRAWLER_OLLAMA_BASE_URL || "http://localhost:11434" };
             break;
           case "nvidia":
             stagehandModel = `openai/${m.startsWith("openai/") ? m.slice(7) : m}`;
-            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, baseURL: baseUrl || "https://integrate.api.nvidia.com/v1" };
+            stagehandModelOpts = { ...apiKey ? { apiKey } : {}, baseURL: baseUrl || process.env.CRAWLER_NVIDIA_BASE_URL || "https://integrate.api.nvidia.com/v1" };
             break;
           case "groq":
             stagehandModel = `groq/${m}`;
@@ -422,15 +448,18 @@ async function crawlProject(opts, onProgress) {
     const preferAnthropic = envModel.includes("claude") || !envModel && !process.env.OPENAI_API_KEY;
     if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || "claude-sonnet-4-6"}`;
+      const defaultAnthropicModel = process.env.CRAWLER_DEFAULT_MODEL ?? "claude-sonnet-4-6";
+      stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || defaultAnthropicModel}`;
       stagehandConfigured = true;
     } else if (process.env.OPENAI_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.OPENAI_API_KEY };
-      stagehandModel = envModel.includes("/") ? envModel : `openai/${envModel || "gpt-4o"}`;
+      const defaultOpenaiModel = process.env.CRAWLER_DEFAULT_MODEL ?? "gpt-4o";
+      stagehandModel = envModel.includes("/") ? envModel : `openai/${envModel || defaultOpenaiModel}`;
       stagehandConfigured = true;
     } else if (process.env.ANTHROPIC_API_KEY) {
       stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
-      stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || "claude-sonnet-4-6"}`;
+      const defaultAnthropicModel2 = process.env.CRAWLER_DEFAULT_MODEL ?? "claude-sonnet-4-6";
+      stagehandModel = envModel.includes("/") ? envModel : `anthropic/${envModel || defaultAnthropicModel2}`;
       stagehandConfigured = true;
     }
   }
@@ -464,7 +493,7 @@ async function crawlProject(opts, onProgress) {
     type: "CRAWL_PROFILE",
     payload: { profile: domainProfile, routeDecision }
   }).catch((err) => console.warn("[crawler] profile evidence save failed:", err?.message ?? err));
-  if (!stagehandConfigured) {
+  if (!stagehandConfigured && !opts.discoveryMode) {
     throw new Error(
       configuredProvider ? `No API key configured for provider "${configuredProvider}". Go to Workspace \u2192 Settings \u2192 AI Provider.` : "No AI provider configured. Go to Workspace \u2192 Settings \u2192 AI Provider to set up an AI provider for browser crawling."
     );
@@ -607,17 +636,17 @@ async function crawlProject(opts, onProgress) {
     verbose: 1,
     logger: stagehandLogger,
     localBrowserLaunchOptions: { cdpUrl, headless: true },
-    domSettleTimeout: 2e3
+    domSettleTimeout: parseInt(process.env.CRAWLER_DOM_SETTLE_TIMEOUT_MS ?? "") || 2e3
   });
   await stagehand.init();
   await onProgress?.(5, "Browser launched");
   const page = await stagehand.resolvePage();
   try {
-    page.setDefaultTimeout(45e3);
+    page.setDefaultTimeout(tuning.defaultTimeoutMs || 45e3);
   } catch {
   }
   try {
-    page.setDefaultNavigationTimeout(3e4);
+    page.setDefaultNavigationTimeout(tuning.navTimeoutMs || 3e4);
   } catch {
   }
   let rawPage = null;
@@ -707,11 +736,91 @@ async function crawlProject(opts, onProgress) {
     const pwContext = playwrightBrowser.contexts()[0];
     if (pwContext) {
       attachNetworkObserver(pwContext, capturedApiEndpoints, (relPath) => xhrDiscoveredPaths.add(relPath));
+      const OAUTH_PROVIDER_RE = /google\.|github\.|microsoft\.|facebook\.|twitter\.|linkedin\.|okta\.|auth0\.|ping\.|onelogin\.|sso\.|oauth\.|\/authorize\?|\/oauth2\//i;
       pwContext.on("page", (popup) => {
-        const popupUrl = popup.url();
-        logger.info({ url: popupUrl }, "[crawler] OAuth popup detected \u2014 waiting for close");
-        popup.waitForEvent("close", { timeout: 12e4 }).then(() => logger.info("[crawler] OAuth popup closed \u2014 SSO flow complete")).catch(() => logger.warn("[crawler] OAuth popup timed out or errored"));
+        popup.waitForURL((u) => !u.startsWith("about:"), { timeout: 1e4 }).then(async () => {
+          const popupUrl = popup.url();
+          if (OAUTH_PROVIDER_RE.test(popupUrl)) {
+            logger.info({ url: popupUrl }, "[crawler] OAuth popup navigated to provider");
+            popup.waitForEvent("close", { timeout: 12e4 }).then(() => logger.info("[crawler] OAuth popup closed \u2014 SSO flow complete")).catch(() => logger.warn("[crawler] OAuth popup timed out or errored"));
+            return;
+          }
+          try {
+            const _popupPath = (() => {
+              try {
+                return new URL(popupUrl).pathname;
+              } catch {
+                return "";
+              }
+            })();
+            if (API_PATH_RE.test(_popupPath) || EXPORT_PATH_RE.test(_popupPath) || DOWNLOAD_EXT_RE.test(_popupPath)) return;
+          } catch {
+          }
+          logger.info({ url: popupUrl }, "[crawler] App popup detected \u2014 capturing");
+          try {
+            await popup.waitForLoadState("domcontentloaded", { timeout: 1e4 }).catch(() => {
+            });
+            await popup.waitForLoadState("networkidle", { timeout: 6e3 }).catch(() => {
+            });
+            const popupTitle = await popup.title().catch(() => "");
+            const popupName = popupTitle || (() => {
+              try {
+                return new URL(popupUrl).pathname.split("/").filter(Boolean).pop() || "Popup";
+              } catch {
+                return "Popup";
+              }
+            })();
+            const popupHash = popupUrl.replace(/[^a-z0-9]/gi, "").slice(-12);
+            const popupShotPath = path.join(opts.screenshotDir ?? "/tmp", `popup-${popupHash}.png`);
+            await waitForScreenshotReady(popup, popup);
+            await popup.screenshot({ path: popupShotPath, fullPage: false }).catch(() => {
+            });
+            const popupScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, popupName, popupUrl, opts.jobId, "LINK_FOLLOW").catch(() => null);
+            if (popupScreen) {
+              const cloudUrl = await uploadToCloud(popupShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(popupShotPath)}`).catch(() => void 0);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, popupScreen.id, popupShotPath, popupScreen.version, "DESKTOP", cloudUrl).catch(() => {
+              });
+              const popupEls = await domFallbackElements(popup, popup).catch(() => []);
+              for (const el of popupEls) {
+                AppBrain.saveElement(opts.tenantId, opts.projectId, popupScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+                });
+              }
+              logger.info({ url: popupUrl, name: popupName, elementCount: popupEls.length }, "[crawler] App popup screen saved");
+            }
+          } catch (popupErr) {
+            logger.warn({ url: popupUrl, err: popupErr?.message }, "[crawler] App popup capture failed \u2014 non-fatal");
+          }
+        }).catch(() => {
+        });
+      });
+      pwContext.on("download", (dl) => {
+        dl.cancel().catch(() => {
+        });
       });
+      try {
+        await pwContext.route("**/*", async (route, request) => {
+          const resourceType = request.resourceType?.() ?? "";
+          if (resourceType !== "document") {
+            await route.continue().catch(() => {
+            });
+            return;
+          }
+          try {
+            const u = new URL(request.url());
+            if (API_PATH_RE.test(u.pathname) || EXPORT_PATH_RE.test(u.pathname) || DOWNLOAD_EXT_RE.test(u.pathname)) {
+              logger.info({ url: request.url() }, "[crawler] route: aborting API/export/download navigation");
+              await route.abort("failed").catch(() => {
+              });
+              return;
+            }
+          } catch {
+          }
+          await route.continue().catch(() => {
+          });
+        });
+      } catch (routeErr) {
+        logger.warn({ err: routeErr?.message }, "[crawler] route interception setup failed \u2014 downloads may occur");
+      }
     }
     const pwContext2 = playwrightBrowser?.contexts()?.[0];
     if (pwContext2) {
@@ -799,6 +908,78 @@ async function crawlProject(opts, onProgress) {
     const homeFingerprintElemCount = await page.evaluate(
       () => document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
     ).catch(() => -1);
+    const preAuthPublicCaptured = /* @__PURE__ */ new Set();
+    if (!opts.discoveryMode) {
+      const _appOrigin = new URL(opts.appUrl).origin;
+      const _norm = (u) => {
+        try {
+          const p = new URL(u);
+          p.hash = "";
+          return p.href.replace(/\/$/, "");
+        } catch {
+          return u;
+        }
+      };
+      const _startNorm = _norm(opts.appUrl);
+      const _authPathRe = /\/(sign.?in|sign.?up|log.?in|register|auth|oauth|sso|forgot.?password|reset.?password)(\/|$|\?)/i;
+      const _skipPathRe = /\/(terms|privacy|legal|cookie-policy|changelog|blog|imprint)([-/]|$)/i;
+      const publicNavLinks = await page.evaluate((originStr) => {
+        return Array.from(document.querySelectorAll("a[href]")).map((a) => {
+          try {
+            const u = new URL(a.getAttribute("href"), window.location.href);
+            return u.origin === originStr ? u.href.split("#")[0] : "";
+          } catch {
+            return "";
+          }
+        }).filter((h, i, arr) => h && arr.indexOf(h) === i).slice(0, 30);
+      }, _appOrigin).catch(() => []);
+      for (const link of publicNavLinks) {
+        if (_norm(link) === _startNorm) continue;
+        try {
+          const { pathname } = new URL(link);
+          if (_authPathRe.test(pathname) || _skipPathRe.test(pathname) || API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
+          await page.goto(link, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => {
+          });
+          await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
+          });
+          const finalUrl = page.url();
+          const finalPath = new URL(finalUrl).pathname;
+          if (_authPathRe.test(finalPath)) continue;
+          if (_norm(finalUrl) === _startNorm) continue;
+          const preAuthElements = await readScreen(page).catch(() => []);
+          if (preAuthElements.length === 0) {
+            logger.info({ url: finalUrl }, "[crawler] pre-auth page has 0 elements \u2014 deferring to authenticated BFS");
+            continue;
+          }
+          const pathParts = finalPath.split("/").filter(Boolean);
+          const screenName = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, finalUrl, opts.jobId, "LINK_FOLLOW").catch(() => null);
+          if (!screen) continue;
+          const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${screen.id}-desktop-v${screen.version}.png`);
+          await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {
+          });
+          const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => void 0);
+          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl).catch(() => {
+          });
+          await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+          });
+          for (const el of preAuthElements) {
+            await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, void 0, void 0, true, void 0, void 0, void 0).catch(() => {
+            });
+          }
+          preAuthPublicCaptured.add(finalUrl);
+          logger.info({ url: finalUrl, screenId: screen.id, elements: preAuthElements.length }, "[crawler] pre-auth public page captured");
+        } catch (err) {
+          logger.warn({ url: link, err: err?.message }, "[crawler] pre-auth public page visit failed \u2014 skipping");
+        }
+      }
+      if (preAuthPublicCaptured.size > 0) {
+        await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 2e4 }).catch(() => {
+        });
+        await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+        });
+      }
+    }
     if (!opts.discoveryMode) {
       try {
         await runAuthDiscoveryPhase(page, opts, new URL(opts.appUrl).origin, onProgress);
@@ -806,6 +987,16 @@ async function crawlProject(opts, onProgress) {
         logger.warn({ err: preAuthErr }, "[crawler] auth-discovery phase failed \u2014 continuing with authenticated crawl");
       }
     }
+    {
+      const worldOk = await page.evaluate(() => true).catch(() => false);
+      if (!worldOk) {
+        logger.warn("[crawler] isolated world corrupted after auth discovery \u2014 re-navigating to recover");
+        await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => {
+        });
+        await page.waitForLoadState("networkidle", { timeout: 6e3 }).catch(() => {
+        });
+      }
+    }
     let didLogin = false;
     if (opts.authToken) {
       await onProgress?.(18, "API token configured \u2014 skipping login");
@@ -819,8 +1010,42 @@ async function crawlProject(opts, onProgress) {
           if (browserCtx) {
             await browserCtx.addCookies(state.cookies);
           }
+          try {
+            const stagehandCtx = page.context?.();
+            if (stagehandCtx && typeof stagehandCtx.addCookies === "function") {
+              await stagehandCtx.addCookies(state.cookies);
+            }
+          } catch {
+          }
         }
         if (state.origins?.length) {
+          const originCtx = playwrightBrowser?.contexts()[0];
+          if (originCtx) {
+            await originCtx.addInitScript((origins) => {
+              for (const entry of origins ?? []) {
+                if (window.location.origin === entry.origin) {
+                  for (const item of entry.localStorage) {
+                    try {
+                      localStorage.setItem(item.name, item.value);
+                    } catch {
+                    }
+                  }
+                }
+              }
+            }, state.origins);
+          }
+          await page.addInitScript((origins) => {
+            for (const entry of origins ?? []) {
+              if (window.location.origin === entry.origin) {
+                for (const item of entry.localStorage) {
+                  try {
+                    localStorage.setItem(item.name, item.value);
+                  } catch {
+                  }
+                }
+              }
+            }
+          }, state.origins);
           await rawPage.addInitScript((origins) => {
             for (const entry of origins ?? []) {
               if (window.location.origin === entry.origin) {
@@ -1024,12 +1249,43 @@ async function crawlProject(opts, onProgress) {
           "nonce",
           "session",
           "sid",
+          // App content/state params — these change the content rendered within the same
+          // page template (a playground example, a search query, a tab selection) but do NOT
+          // represent structurally distinct pages. Not stripping them causes URL explosion:
+          // every unique ?cmd=DSN-XXXXX&example=... or ?q=foo is queued as a new screen.
+          "cmd",
+          "example",
+          "query",
+          "q",
+          "search",
+          "keyword",
+          "keywords",
+          "tab",
+          "view",
+          "mode",
+          "sort",
+          "order",
+          "direction",
+          "orderby",
+          "page",
+          "p",
+          "offset",
+          "limit",
+          "per_page",
+          "size",
+          "rows",
+          // Export format params — same endpoint, different output format, not a new screen
+          "format",
+          "output",
+          "export_format",
+          "download_format",
+          "fileformat",
+          "file_format",
           // WooCommerce / e-commerce action params
           "add-to-cart",
           "add_to_wishlist",
           "quantity",
           "wc-ajax",
-          "orderby",
           "min_price",
           "max_price",
           "removed_item",
@@ -1097,9 +1353,11 @@ async function crawlProject(opts, onProgress) {
     };
     const gitInfo = getGitInfo();
     const visited = /* @__PURE__ */ new Map();
+    const crawlWarnings = [];
     if (opts.skipUrls?.length) {
       for (const u of opts.skipUrls) visited.set(canonicalize(u), "skipped");
     }
+    for (const u of preAuthPublicCaptured) visited.set(canonicalize(u), "pre-auth");
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0;
     let lastEmittedPct = 20;
@@ -1133,7 +1391,7 @@ async function crawlProject(opts, onProgress) {
     ];
     const queued = new Set(seedUrls.map(canonicalize));
     const queue = new CrawlFrontier(opts.traversalStrategy ?? "BFS", seedUrls, opts.bestFirstKeywords, opts.urlFilter);
-    const deduplicator = new ContentDeduplicator(0.88);
+    const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? "") || 0.88);
     const deadLinkTracker = new DeadLinkTracker();
     const redirectRegistry = new RedirectRegistry();
     if (!selective && resumeUrls.length === 0) {
@@ -1143,6 +1401,7 @@ async function crawlProject(opts, onProgress) {
         const orphanIdSet = new Set(
           (orphanReport?.screens ?? []).filter((s) => s.classification !== "REACHABLE").map((s) => s.id)
         );
+        const seenReseedCanon = new Set(queued);
         const priorScreens = (priorMap?.screens ?? []).filter((s) => s.status !== "PENDING_REMOVAL").filter((s) => {
           const u = s.url;
           if (!u) return false;
@@ -1151,11 +1410,15 @@ async function crawlProject(opts, onProgress) {
             const stripWww = (h) => h.replace(/^www\./i, "").toLowerCase();
             if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
             if (isGeneratedDynamicPath(pathname)) return false;
+            if (!isUsefulNavigationUrl(u, startUrl)) return false;
+            const canon = canonicalize(u);
+            if (seenReseedCanon.has(canon)) return false;
+            seenReseedCanon.add(canon);
             return true;
           } catch {
             return false;
           }
-        }).filter((s) => !queued.has(canonicalize(s.url)));
+        });
         let orphanSeedCount = 0;
         for (const s of priorScreens) {
           const u = s.url;
@@ -1213,6 +1476,7 @@ async function crawlProject(opts, onProgress) {
             try {
               const u = new URL(sUrl);
               if (u.origin !== sitemapOrigin) continue;
+              if (API_PATH_RE.test(u.pathname) || EXPORT_PATH_RE.test(u.pathname) || DOWNLOAD_EXT_RE.test(u.pathname)) continue;
               await page.goto(sUrl, { waitUntil: "domcontentloaded", timeout: 12e3 });
               await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
               });
@@ -1268,16 +1532,20 @@ async function crawlProject(opts, onProgress) {
     const discovered = [];
     const performanceMetrics = [];
     const seenDomHashes = new Set(opts.knownDomHashes ?? []);
-    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
+    const skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA ? process.env.CRAWL_SKIP_PATH_EXTRA.split(",").map((p) => p.trim()).filter(Boolean) : [];
+    const SKIP_PATH = skipPathExtra.length > 0 ? new RegExp(`/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap|${skipPathExtra.join("|")})([-/]|$)`, "i") : /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
     const segIdx = opts.segmentIndex ?? 1;
     const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
     let avgNavMs = 0;
     let navCount = 0;
     let aiErrorStreak = 0;
     let aiSkipCount = 0;
+    const aiMinElements = parseInt(process.env.CRAWL_AI_LINK_MIN_ELEMENTS ?? "") || 5;
     let aiRateLimitWarned = false;
     let totalAttemptsThisSegment = 0;
     const maxAttempts = Math.max(segmentSize * 4, 200);
+    let loginWallRedirectCount = 0;
+    const LOGIN_WALL_ABORT_THRESHOLD = Math.max(8, Math.ceil(segmentSize * 0.4));
     let pendingSessionRestore = null;
     const CRAWL_PAGE_PARALLELISM = Math.max(1, parseInt(process.env.CRAWL_PAGE_PARALLELISM ?? "5", 10));
     async function runLiteWorker(workerPage) {
@@ -1291,18 +1559,42 @@ async function crawlProject(opts, onProgress) {
         try {
           const { pathname } = new URL(url);
           if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+          if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
         } catch {
           continue;
         }
         visited.set(canonUrl, "_pending_");
         try {
-          const navErr = await workerPage.goto(url, { waitUntil: "domcontentloaded", timeout: 15e3 }).then(() => null, (e) => e);
+          const navErr = await workerPage.goto(url, { waitUntil: "domcontentloaded", timeout: tuning.navTimeoutMs }).then(() => null, (e) => e);
           if (navErr && /timeout/i.test(String(navErr?.message ?? navErr))) {
             await workerPage.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(() => {
             });
           }
           await workerPage.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
           });
+          await waitForStableDOM(workerPage, 6e3, 400).catch(() => {
+          });
+          await dismissOverlays(workerPage).catch(() => {
+          });
+          const lwCurrentUrl = workerPage.url();
+          const lwPath = (() => {
+            try {
+              return new URL(lwCurrentUrl).pathname.toLowerCase();
+            } catch {
+              return "";
+            }
+          })();
+          const lwIsAuthWall = /^\/(login|signin|sign-in|auth|authenticate|session\/new|account\/login|user\/login)/.test(lwPath) && lwCurrentUrl !== url;
+          const lwHasLoginForm = lwIsAuthWall ? false : await workerPage.evaluate(() => {
+            const inputs = document.querySelectorAll('input[type="password"]');
+            return inputs.length > 0;
+          }).catch(() => false);
+          if (lwIsAuthWall) {
+            logger.warn({ url, landed: lwCurrentUrl }, "[lite-worker] auth wall detected \u2014 queuing for main worker re-auth");
+            visited.set(canonUrl, "_auth_wall_");
+            if (!revisitQueue.includes(url)) revisitQueue.push(url);
+            continue;
+          }
           if (homeFingerprintElemCount >= 0 && url !== startUrl) {
             const wTitle = await workerPage.title().catch(() => "");
             if (wTitle === homeFingerprintTitle && wTitle !== "") {
@@ -1321,31 +1613,169 @@ async function crawlProject(opts, onProgress) {
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
           visited.set(canonUrl, screen.id);
           const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${screen.id}-desktop-v${screen.version}.png`);
+          let liteElements = await domFallbackElements(workerPage).catch(() => []);
+          if (liteElements.length === 0) {
+            liteElements = await workerPage.evaluate(() => {
+              const sel = 'button,a[href],input,select,textarea,[role],[tabindex="0"],[onclick],[data-action]';
+              return Array.from(document.querySelectorAll(sel)).filter((el) => el.offsetParent !== null).slice(0, 150).map((el) => ({
+                meaning: el.getAttribute("aria-label") || el.textContent?.trim().slice(0, 100) || el.tagName.toLowerCase(),
+                role: el.getAttribute("role") || el.tagName.toLowerCase(),
+                expectedData: null,
+                notes: null,
+                boundingRect: null,
+                ariaState: null,
+                parentLandmark: null
+              }));
+            }).catch(() => []);
+          }
           const [elements] = await Promise.all([
-            domFallbackElements(workerPage).catch(() => []),
+            Promise.resolve(liteElements),
             workerPage.screenshot({ path: shotPath, fullPage: true }).then(async () => {
               const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
               return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl);
             }).catch(() => {
             })
           ]);
+          const _navEls = await workerPage.evaluate(() => {
+            if (typeof globalThis.__name === "undefined") globalThis.__name = (fn) => fn;
+            const seen = /* @__PURE__ */ new Set();
+            const items = [];
+            function isV(el) {
+              const s = window.getComputedStyle(el);
+              return s.display !== "none" && s.visibility !== "hidden" && s.opacity !== "0";
+            }
+            document.querySelectorAll('nav a, [role="navigation"] a, header a, footer nav a, [aria-label*="navigation" i] a').forEach((a) => {
+              if (!isV(a)) return;
+              const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
+              const href = a.href ?? "";
+              if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
+              let landmark = "Site Navigation";
+              let cur = a.parentElement;
+              while (cur) {
+                const t = cur.tagName?.toLowerCase();
+                const r = cur.getAttribute?.("role")?.toLowerCase();
+                if (t === "header" || r === "banner") {
+                  landmark = "Header";
+                  break;
+                }
+                if (t === "footer" || r === "contentinfo") {
+                  landmark = "Footer";
+                  break;
+                }
+                if (t === "nav" || r === "navigation") {
+                  landmark = cur.getAttribute("aria-label")?.trim() || "Site Navigation";
+                  break;
+                }
+                cur = cur.parentElement;
+              }
+              const key = `nav|${label.toLowerCase()}|${href}`;
+              if (!seen.has(key)) {
+                seen.add(key);
+                items.push({ meaning: label, role: "nav-link", notes: JSON.stringify({ href: href.slice(0, 150), landmark }) });
+              }
+            });
+            let fc = 0;
+            document.querySelectorAll('footer a, [role="contentinfo"] a, [class*="footer" i] a').forEach((a) => {
+              if (fc >= 20 || !isV(a)) return;
+              const label = (a.getAttribute("aria-label") || a.textContent?.replace(/\s+/g, " ").trim() || "").slice(0, 60);
+              const href = a.href ?? "";
+              if (!label || !href) return;
+              const key = `footer|${label.toLowerCase()}|${href}`;
+              if (!seen.has(key)) {
+                seen.add(key);
+                items.push({ meaning: label, role: "footer-link", notes: JSON.stringify({ href: href.slice(0, 150) }) });
+                fc++;
+              }
+            });
+            return items;
+          }).catch(() => []);
+          const allElements = [
+            ...elements,
+            ..._navEls.map((ne) => ({ ...ne, expectedData: void 0, boundingRect: void 0, ariaState: void 0, parentLandmark: void 0 }))
+          ];
+          if (allElements.length > 0) {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+            });
+            for (let _ei = 0; _ei < allElements.length; _ei += 4) {
+              const _batch = allElements.slice(_ei, _ei + 4);
+              await Promise.all(_batch.map(
+                (el) => AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+                })
+              ));
+            }
+            logger.info({ url, screenId: screen.id, elements: elements.length, navLinks: _navEls.length }, "[lite-worker] screen captured");
+          } else if (!revisitQueue.includes(url)) {
+            logger.warn({ url, screenId: screen.id }, "[lite-worker] 0 elements \u2014 queued for P3a revisit");
+            revisitQueue.push(url);
+          }
+          if (screen?.id) {
+            const [shadowRes, a11yRes, swRes] = await Promise.race([
+              Promise.allSettled([
+                crawlShadowDom(workerPage),
+                crawlAccessibilityTree(workerPage),
+                inspectServiceWorker(workerPage)
+              ]),
+              new Promise(
+                (resolve) => setTimeout(() => resolve([
+                  { status: "rejected", reason: "timeout" },
+                  { status: "rejected", reason: "timeout" },
+                  { status: "rejected", reason: "timeout" }
+                ]), 15e3)
+              )
+            ]);
+            const litePayload = { screenId: screen.id, url };
+            if (shadowRes.status === "fulfilled" && shadowRes.value && shadowRes.value.shadowHostCount > 0)
+              litePayload.shadowDom = shadowRes.value;
+            if (a11yRes.status === "fulfilled" && a11yRes.value)
+              litePayload.accessibilityTree = a11yRes.value;
+            if (swRes.status === "fulfilled" && swRes.value.supported)
+              litePayload.serviceWorker = swRes.value;
+            if (Object.keys(litePayload).length > 2) {
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                type: "PER_SCREEN_PROBES",
+                payload: litePayload
+              }).catch(() => {
+              });
+            }
+          }
           const nextLinks = await workerPage.evaluate((originStr) => {
-            return Array.from(document.querySelectorAll("a[href]")).map((a) => {
+            return Array.from(document.querySelectorAll("a[href],[data-href]")).map((a) => {
+              const raw = a.getAttribute("href") || a.getAttribute("data-href") || "";
               try {
-                return new URL(a.getAttribute("href"), window.location.href).href;
+                return new URL(raw, window.location.href).href;
               } catch {
                 return "";
               }
-            }).filter((h) => h && new URL(h).origin === originStr && !h.includes("#")).slice(0, 200);
+            }).filter((h) => {
+              if (!h) return false;
+              try {
+                const u = new URL(h);
+                if (u.origin !== originStr) return false;
+                if (u.hash && !u.hash.startsWith("#/") && !u.hash.startsWith("#!/")) return false;
+                return true;
+              } catch {
+                return false;
+              }
+            }).slice(0, 200);
           }, origin).catch(() => []);
           for (const link of nextLinks) {
+            if (!isUsefulNavigationUrl(link, origin)) continue;
             const cLink = canonicalize(link);
-            if (!visited.has(cLink)) {
+            if (!visited.has(cLink) && !queued.has(cLink)) {
+              queued.add(cLink);
               queue.push(link);
               edges.push({ fromUrl: url, toUrl: link });
             }
           }
           discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+          const lwEffMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+          maxSeenTotal = Math.max(maxSeenTotal, lwEffMax);
+          const lwRawPct = 20 + Math.round(Math.min(57, visited.size / Math.max(1, maxSeenTotal) * 57));
+          const lwPct = Math.max(lastEmittedPct, lwRawPct);
+          lastEmittedPct = lwPct;
+          await onProgress?.(lwPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`).catch(() => {
+          });
         } catch (err) {
           logger.warn({ url, err: err?.message ?? String(err) }, "[lite-worker] URL processing failed");
           discovered.push({ url, error: err?.message ?? String(err) });
@@ -1353,7 +1783,7 @@ async function crawlProject(opts, onProgress) {
       }
     }
     const liteWorkerPromises = [];
-    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser) {
+    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser && !opts.discoveryMode) {
       const pwCtx = playwrightBrowser.contexts()[0];
       for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
         const wp = await pwCtx.newPage();
@@ -1361,7 +1791,7 @@ async function crawlProject(opts, onProgress) {
           Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
         }).catch(() => {
         });
-        wp.setDefaultTimeout(45e3);
+        wp.setDefaultTimeout(tuning.defaultTimeoutMs || 45e3);
         liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {
         })));
       }
@@ -1386,9 +1816,10 @@ async function crawlProject(opts, onProgress) {
       }
       const url = queue.shift();
       const canonUrl = canonicalize(url);
-      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size + 1}/${segmentSize}` }, "[crawler] visiting URL");
       const apiObservationStartIndex = capturedApiEndpoints.length;
       if (visited.has(canonUrl)) continue;
+      visited.set(canonUrl, "_pending_");
+      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size}/${segmentSize}` }, "[crawler] visiting URL");
       if (robotsCache && !await robotsCache.isAllowed(url)) {
         logger.info({ url }, "[crawler] robots.txt: skipping disallowed URL");
         continue;
@@ -1403,6 +1834,11 @@ async function crawlProject(opts, onProgress) {
           console.log(`[crawler] skip generated dynamic route: ${url}`);
           continue;
         }
+        if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) {
+          console.log(`[crawler] skip API/export/download path: ${url}`);
+          visited.set(canonUrl, "_skipped_api_");
+          continue;
+        }
       } catch {
       }
       try {
@@ -1481,7 +1917,7 @@ async function crawlProject(opts, onProgress) {
               logger.info({ url }, "[crawler] temporarily cleared session for public auth route");
             }
           }
-          const navErr15 = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 15e3 }).then(() => null, (e) => e);
+          const navErr15 = await page.goto(url, { waitUntil: "domcontentloaded", timeout: tuning.navTimeoutMs }).then(() => null, (e) => e);
           if (navErr15) {
             if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
               await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch((navErr) => {
@@ -1493,14 +1929,19 @@ async function crawlProject(opts, onProgress) {
           }
           rawPage.off("response", onRedirectResponse);
           if (redirectHops.length > 0) {
+            const isCircularRedirect = new Set(redirectHops.map((h) => h.from)).size < redirectHops.length;
             redirectRegistry.register({
               originalUrl: url,
               finalUrl: page.url(),
               hops: redirectHops.map((h, i) => ({ ...h, durationMs: i === 0 ? Date.now() - navStart : 0 })),
-              isCircular: new Set(redirectHops.map((h) => h.from)).size < redirectHops.length,
+              isCircular: isCircularRedirect,
               hopCount: redirectHops.length,
               totalDurationMs: Date.now() - navStart
             });
+            if (isCircularRedirect) {
+              logger.warn({ url, hops: redirectHops.map((h) => h.from) }, "[crawler] circular redirect detected \u2014 skipping URL");
+              continue;
+            }
           }
           const navMs = Date.now() - navStart;
           navCount++;
@@ -1510,9 +1951,36 @@ async function crawlProject(opts, onProgress) {
           console.warn(`[crawler] connection failed for ${url} \u2014 skipping (chrome-error page)`);
           continue;
         }
-        await page.waitForLoadState("load", { timeout: 8e3 }).catch(() => {
+        await (rawPage ?? page).waitForLoadState("load", { timeout: 8e3 }).catch(() => {
         });
-        await waitForStableDOM(page);
+        await waitForStableDOM(page, 8e3, 400, rawPage ?? void 0);
+        const sparseCheck = await (rawPage ?? page).evaluate(
+          () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[aria-label]').length
+        ).catch(() => 99);
+        if (sparseCheck < 15) {
+          await (rawPage ?? page).waitForLoadState("networkidle", { timeout: 6e3 }).catch(() => {
+          });
+          await waitForStableDOM(page, 4e3, 300, rawPage ?? void 0);
+        }
+        try {
+          const spinnerPage = rawPage ?? page;
+          if (typeof spinnerPage.waitForFunction === "function") {
+            await spinnerPage.waitForFunction(() => {
+              const body = document.body;
+              if (!body) return true;
+              const text = body.innerText?.toLowerCase() ?? "";
+              if (text.includes("checking authentication") || text.includes("checking auth") || text.includes("authenticating") || text.includes("verifying session") || text.includes("please wait") || text.includes("loading data") || text.includes("loading") && text.length < 200) return false;
+              const busyEl = body.querySelector('[aria-busy="true"]');
+              if (busyEl && busyEl.offsetParent !== null) return false;
+              if (body.querySelector('[data-loading="true"],[data-pending="true"],[data-fetching="true"]')) return false;
+              if (body.querySelector("progress:not([value])")) return false;
+              const skeleton = body.querySelector(".skeleton,.skeleton-loader,.shimmer,.pulse-loader,.content-loader");
+              if (skeleton && skeleton.offsetParent !== null) return false;
+              return true;
+            }, { timeout: 12e3 });
+          }
+        } catch {
+        }
         if (!opts.discoveryMode) {
           try {
             const appModalVisible = await page.evaluate(() => {
@@ -1529,7 +1997,7 @@ async function crawlProject(opts, onProgress) {
               const modalShotPath = path.join(opts.screenshotDir ?? "/tmp", `modal-${modalHash}.png`);
               await page.screenshot({ path: modalShotPath, fullPage: false }).catch(() => {
               });
-              const modalName = await page.evaluate(() => {
+              const modalName = await (rawPage ?? page).evaluate(() => {
                 const h = document.querySelector('[role="dialog"] h1,[role="dialog"] h2,[role="dialog"] [role="heading"]');
                 return h?.textContent?.trim() ?? "Modal";
               }).catch(() => "Modal");
@@ -1537,13 +2005,18 @@ async function crawlProject(opts, onProgress) {
               if (modalScreen) {
                 await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, "DESKTOP").catch(() => {
                 });
-                logger.info({ url, name: modalName }, "[crawler] captured app modal before dismiss");
+                const modalElements = await domFallbackElements(page, rawPage ?? void 0).catch(() => []);
+                for (const el of modalElements) {
+                  AppBrain.saveElement(opts.tenantId, opts.projectId, modalScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+                  });
+                }
+                logger.info({ url, name: modalName, elementCount: modalElements.length }, "[crawler] captured app modal before dismiss");
               }
             }
           } catch {
           }
           await dismissOverlays(page);
-          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }).catch(() => {
+          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }, rawPage ?? void 0).catch(() => {
           });
         }
         if (cdpSession?.off) cdpSession.off("Network.responseReceived", secHeadersCdp);
@@ -1563,6 +2036,22 @@ async function crawlProject(opts, onProgress) {
         const weWantedThisPage = BFS_ERROR_PATH.test(new URL(url).pathname) || BFS_LOGIN_PATH.test(new URL(url).pathname);
         const redirectedToError = !weWantedThisPage && (BFS_ERROR_PATH.test(new URL(landedUrl).pathname) || BFS_LOGIN_PATH.test(new URL(landedUrl).pathname));
         if (redirectedToError) {
+          if (BFS_LOGIN_PATH.test(new URL(landedUrl).pathname)) {
+            loginWallRedirectCount++;
+            if (!opts.credentials && loginWallRedirectCount >= LOGIN_WALL_ABORT_THRESHOLD) {
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                type: "AUTH_FAILURE",
+                payload: {
+                  url,
+                  landed: landedUrl,
+                  reason: `Login wall detected: ${loginWallRedirectCount} consecutive URLs redirected to login. Add credentials in project settings to crawl authenticated pages.`
+                }
+              }).catch(() => {
+              });
+              throw new Error(`[AUTH_FAILURE] Login wall detected: ${loginWallRedirectCount} URLs redirected to the login page. Add credentials in project settings and retry the crawl.`);
+            }
+          }
           logger.warn({ requested: url, landed: landedUrl }, "[crawler] Mid-crawl redirect to auth/error \u2014 attempting re-auth");
           let reauthed = false;
           if (opts.credentials) {
@@ -1579,9 +2068,9 @@ async function crawlProject(opts, onProgress) {
               logger.info({ url }, "[crawler] Re-auth succeeded \u2014 retrying URL");
               await page.goto(url, { waitUntil: "domcontentloaded" }).catch(() => {
               });
-              await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+              await (rawPage ?? page).waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
               });
-              await waitForStableDOM(page);
+              await waitForStableDOM(page, 8e3, 400, rawPage ?? void 0);
               const afterReauthUrl = page.url();
               if (BFS_ERROR_PATH.test(new URL(afterReauthUrl).pathname) || BFS_LOGIN_PATH.test(new URL(afterReauthUrl).pathname)) {
                 logger.warn({ url }, "[crawler] Still redirecting after re-auth \u2014 skipping this URL");
@@ -1598,6 +2087,10 @@ async function crawlProject(opts, onProgress) {
               throw new Error("[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.");
             }
           } else {
+            if (opts.discoveryMode) {
+              logger.info({ url, landed: landedUrl }, "[discover] skipping auth-required URL \u2014 no credentials, continuing BFS");
+              continue;
+            }
             logger.warn("[crawler] Session expired mid-crawl, no credentials to recover \u2014 skipping URL");
             AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
               jobId: opts.jobId,
@@ -1611,6 +2104,7 @@ async function crawlProject(opts, onProgress) {
         if (!redirectedToError && landedUrl !== url) {
           logger.info({ requested: url, landed: landedUrl }, "[crawler] navigated with redirect");
         }
+        if (!redirectedToError) loginWallRedirectCount = 0;
         if (opts.discoveryMode) {
           visited.set(canonicalize(url), url);
           await (rawPage ?? page).waitForLoadState("networkidle", { timeout: 1e4 }).catch(() => {
@@ -1749,7 +2243,7 @@ async function crawlProject(opts, onProgress) {
           continue;
         }
         const preDomHash = await computeDomHash(page);
-        if (seenDomHashes.has(preDomHash)) {
+        if (preDomHash !== "" && seenDomHashes.has(preDomHash)) {
           console.log(`[crawler] skip ${url} \u2014 duplicate DOM structure (same content as prior page)`);
           continue;
         }
@@ -1862,7 +2356,7 @@ async function crawlProject(opts, onProgress) {
           } catch {
           }
           try {
-            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: 2e3 }).catch(() => null);
+            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
             if (activeNav) {
               const t = activeNav.trim();
               if (t.length > 1 && t.length < 60 && t.toLowerCase() !== brandName) return t;
@@ -1881,7 +2375,7 @@ async function crawlProject(opts, onProgress) {
               ".page-header h2"
             ];
             for (const sel of selectors) {
-              const h = await page.locator(sel).first().textContent({ timeout: 2e3 }).catch(() => null);
+              const h = await page.locator(sel).first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
               if (!h) continue;
               const t = h.trim();
               if (t.length < 2 || t.length > 80) continue;
@@ -1971,21 +2465,24 @@ async function crawlProject(opts, onProgress) {
         if (cachedLinks) {
           logger.info({ url, cachedCount: cachedLinks.length }, "[crawler] link-discovery: using cached recipe \u2014 AI will still run and merge");
         }
-        const elapsedDiscoverySec = (Date.now() - crawlLoopStart) / 1e3;
-        const discoveryPct = Math.round(5 + Math.min(15, elapsedDiscoverySec / 60 * 15));
-        const preCapturePct = Math.max(lastEmittedPct, discoveryPct);
-        lastEmittedPct = preCapturePct;
-        await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} \xB7 ${(() => {
-          try {
-            return new URL(url).pathname || "/";
-          } catch {
-            return "/";
-          }
-        })()}`);
+        {
+          const effMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+          maxSeenTotal = Math.max(maxSeenTotal, effMax);
+          const rawPrePct = 20 + Math.round(Math.min(57, visited.size / Math.max(1, maxSeenTotal) * 57));
+          const preCapturePct = Math.max(lastEmittedPct, rawPrePct);
+          lastEmittedPct = preCapturePct;
+          await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} \xB7 ${(() => {
+            try {
+              return new URL(url).pathname || "/";
+            } catch {
+              return "/";
+            }
+          })()}`);
+        }
+        await waitForScreenshotReady(page, rawPage ?? void 0);
         const preDomCount = await page.evaluate(
           () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
         ).catch(() => 0);
-        await waitForScreenshotReady(page);
         await onProgress?.(lastEmittedPct, `Capturing ${(() => {
           try {
             return new URL(url).pathname || "/";
@@ -1993,8 +2490,8 @@ async function crawlProject(opts, onProgress) {
             return "/";
           }
         })()}`);
-        const pageDpr = await page.evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);
-        const elements = await extractElements(stagehand, page, elementLLM ?? void 0).catch((err) => {
+        const pageDpr = await (rawPage ?? page).evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);
+        const elements = await extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch((err) => {
           console.error("[crawler] extractElements failed:", err);
           return [];
         });
@@ -2045,8 +2542,8 @@ async function crawlProject(opts, onProgress) {
           }).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
-          // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
-          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0 || preDomCount < 5).then((freshLinks) => {
+          // Note: preDomCount < aiMinElements guard removed — it was backwards: sparse pages (SPAs) need AI most.
+          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).then((freshLinks) => {
             if (aiSkipCount > 0) aiSkipCount--;
             aiErrorStreak = 0;
             const merged = (cachedLinks ? Array.from(/* @__PURE__ */ new Set([...cachedLinks, ...freshLinks])) : freshLinks).filter((href) => isUsefulNavigationUrl(href, origin));
@@ -2079,7 +2576,7 @@ async function crawlProject(opts, onProgress) {
             }
             return fallback;
           }),
-          readScreen(page).catch(() => [])
+          readScreen(rawPage ?? page).catch(() => [])
         ]);
         await page.evaluate(() => {
           const container = document.getElementById("__zeta_element_highlights");
@@ -2100,6 +2597,104 @@ async function crawlProject(opts, onProgress) {
           }
         }).catch(() => {
         });
+        {
+          const safeContext = typeof rawPage?.context === "function" ? rawPage.context() : typeof page?.context === "function" ? page.context() : playwrightBrowser?.contexts()[0] ?? null;
+          const roTimeout = new Promise(
+            (resolve) => setTimeout(() => resolve([
+              { status: "rejected", reason: "probe-timeout" },
+              { status: "rejected", reason: "probe-timeout" },
+              { status: "rejected", reason: "probe-timeout" },
+              { status: "rejected", reason: "probe-timeout" },
+              { status: "rejected", reason: "probe-timeout" }
+            ]), 25e3)
+          );
+          const [nestedCtxRes, shadowRes, a11yRes, perfRes, swRes] = await Promise.race([
+            Promise.allSettled([
+              crawlNestedContexts(rawPage ?? page, safeContext, url),
+              crawlShadowDom(rawPage ?? page),
+              crawlAccessibilityTree(rawPage ?? page),
+              collectPerfMetrics(rawPage ?? page, url),
+              inspectServiceWorker(rawPage ?? page)
+            ]),
+            roTimeout
+          ]);
+          const roPayload = { screenId: screen.id, url };
+          if (nestedCtxRes.status === "fulfilled" && nestedCtxRes.value && nestedCtxRes.value.totalContextsFound > 0)
+            roPayload.nestedContexts = nestedCtxRes.value;
+          if (shadowRes.status === "fulfilled" && shadowRes.value && shadowRes.value.shadowHostCount > 0)
+            roPayload.shadowDom = shadowRes.value;
+          if (a11yRes.status === "fulfilled" && a11yRes.value)
+            roPayload.accessibilityTree = a11yRes.value;
+          if (perfRes.status === "fulfilled" && perfRes.value.requestCount > 0)
+            roPayload.performance = perfRes.value;
+          if (swRes.status === "fulfilled" && swRes.value.supported)
+            roPayload.serviceWorker = swRes.value;
+          if (Object.keys(roPayload).length > 2) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId,
+              type: "PER_SCREEN_PROBES",
+              payload: roPayload
+            }).catch(() => {
+            });
+          }
+          if (screen?.id && opts.interactive !== false && safeContext) {
+            void (async () => {
+              let probePage = null;
+              try {
+                const openProbePages = safeContext.pages().length;
+                if (openProbePages > 8) {
+                  logger.warn({ url, openProbePages }, "[crawler] probe page cap reached \u2014 skipping mutating probes for this screen");
+                  return;
+                }
+                probePage = await safeContext.newPage();
+                await probePage.goto(url, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => {
+                });
+                await probePage.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+                });
+                await waitForStableDOM(probePage, 4e3, 400).catch(() => {
+                });
+                const muTimeout = new Promise(
+                  (resolve) => setTimeout(() => resolve([
+                    { status: "rejected", reason: "probe-timeout" },
+                    { status: "rejected", reason: "probe-timeout" },
+                    { status: "rejected", reason: "probe-timeout" },
+                    { status: "rejected", reason: "probe-timeout" }
+                  ]), 6e4)
+                );
+                const [dropEnumRes, crudRes, filterRes, advancedRes] = await Promise.race([
+                  Promise.allSettled([
+                    enumerateDropdowns(probePage),
+                    probeCrudFlows(probePage, name),
+                    crawlFiltersAndPagination(probePage),
+                    probeAdvancedUI(probePage)
+                  ]),
+                  muTimeout
+                ]);
+                const muPayload = { screenId: screen.id, url };
+                if (dropEnumRes.status === "fulfilled" && dropEnumRes.value.totalOptionsFound > 0)
+                  muPayload.dropdowns = dropEnumRes.value;
+                if (crudRes.status === "fulfilled" && (crudRes.value.flows?.length > 0 || crudRes.value.toastsObserved?.length > 0))
+                  muPayload.crudFlows = crudRes.value;
+                if (filterRes.status === "fulfilled" && filterRes.value.filters?.length > 0)
+                  muPayload.filtersAndPagination = filterRes.value;
+                if (advancedRes.status === "fulfilled" && advancedRes.value && (advancedRes.value.datePickers?.length > 0 || advancedRes.value.sliders?.length > 0 || advancedRes.value.typeaheads?.length > 0 || advancedRes.value.dragDrop?.library !== "none"))
+                  muPayload.advancedUI = advancedRes.value;
+                if (Object.keys(muPayload).length > 2) {
+                  AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                    jobId: opts.jobId,
+                    type: "PER_SCREEN_PROBES_MUTATING",
+                    payload: muPayload
+                  }).catch(() => {
+                  });
+                }
+              } catch {
+              } finally {
+                await probePage?.close().catch(() => {
+                });
+              }
+            })();
+          }
+        }
         try {
           const domContent = await capturePageHtml(page);
           if (!domContent) {
@@ -2341,159 +2936,196 @@ ${snippet}`,
           })();
         }
         const linkMappings = await extractLinkMappings(page, origin);
-        await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
-        });
-        for (let _ei = 0; _ei < elements.length; _ei += 4) {
-          const _batch = elements.slice(_ei, _ei + 4);
-          await Promise.all(_batch.map(async (el) => {
-            const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark);
-            const label = el.meaning;
-            const role = el.role ?? "unknown";
-            const selectorHint = atByRole.get(role) ?? role;
-            const selectors = [
-              { type: "role-label", value: `${role}:${label}`, confidence: 0.9 },
-              { type: "aria-label", value: label, confidence: 0.85 },
-              { type: "tag-hint", value: selectorHint, confidence: 0.6 }
-            ];
-            AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
-          }));
-        }
-        for (const lm of linkMappings) {
-          const safety = assessActionSafety(lm.label, domainProfile);
-          if (!safety.allowed) {
-            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
-              screenId: screen.id,
-              jobId: opts.jobId,
-              type: "ACTION_BLOCKED_BY_SAFETY_POLICY",
-              payload: { label: lm.label, href: lm.href, decision: safety, profile: domainProfile.industry }
-            }).catch(() => {
-            });
-            continue;
+        const postExtractUrl = (() => {
+          try {
+            return page.url();
+          } catch {
+            return url;
           }
-          pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
-          AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
-            domain: new URL(url).hostname,
-            purpose: `navigate:${lm.label}`,
-            fingerprint: sha256(`${canonicalize(url)}:${lm.label}:${canonicalize(lm.href)}`),
-            steps: [{ action: "click", label: lm.label, targetUrl: lm.href, fromUrl: url }],
-            successRate: 0.5
+        })();
+        const postExtractParsed = (() => {
+          try {
+            return new URL(postExtractUrl);
+          } catch {
+            return new URL(url);
+          }
+        })();
+        const driftedToAuthWall = !weWantedThisPage && (BFS_LOGIN_PATH.test(postExtractParsed.pathname) || BFS_ERROR_PATH.test(postExtractParsed.pathname));
+        if (driftedToAuthWall) {
+          logger.warn({ url, postExtractUrl }, "[crawler] Page drifted to auth wall during element extraction \u2014 skipping element save to prevent login-page element contamination");
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            screenId: screen.id,
+            jobId: opts.jobId,
+            type: "AUTH_FAILURE",
+            payload: { url, landed: postExtractUrl, reason: "Session expired during element extraction. Re-capture session and re-crawl this page." }
           }).catch(() => {
           });
-          if (!selective) {
-            try {
-              const targetPath = new URL(lm.href).pathname;
-              const canonLmHref = canonicalize(lm.href);
-              if (!SKIP_PATH.test(targetPath) && isUsefulNavigationUrl(lm.href, origin) && !queued.has(canonLmHref) && !visited.has(canonLmHref) && queued.size < maxScreens * 2) {
-                if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
-                queued.add(canonLmHref);
-                queue.push(lm.href, { parentElements: elements.length });
-                edges.push({ fromUrl: url, toUrl: lm.href });
-                logger.info({ fromUrl: url, newUrl: lm.href, label: lm.label, queueSize: queue.length }, "[crawler] anchor mapping added to queue");
-              }
-            } catch {
+        } else {
+          if (elements.length === 0) {
+            logger.warn({ url, screenId: screen.id }, "[crawler] extractElements returned 0 \u2014 preserving existing elements to avoid data loss");
+            if (!revisitQueue.includes(url)) {
+              revisitQueue.push(url);
+              logger.info({ url }, "[crawler] 0-element page queued for re-visit with extended hydration wait");
+            }
+          } else {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+            });
+            for (let _ei = 0; _ei < elements.length; _ei += 4) {
+              const _batch = elements.slice(_ei, _ei + 4);
+              await Promise.all(_batch.map(async (el) => {
+                const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark);
+                const label = el.meaning;
+                const role = el.role ?? "unknown";
+                const selectorHint = atByRole.get(role) ?? role;
+                const selectors = [
+                  { type: "role-label", value: `${role}:${label}`, confidence: 0.9 },
+                  { type: "aria-label", value: label, confidence: 0.85 },
+                  { type: "tag-hint", value: selectorHint, confidence: 0.6 }
+                ];
+                AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+              }));
             }
           }
-        }
-        if (!selective) {
-          for (const href of nextLinks) {
-            edges.push({ fromUrl: url, toUrl: href });
-            try {
-              const hrefParams = new URL(href).searchParams;
-              const actionVal = hrefParams.get("action") ?? "";
-              if (/^yith-|^wc-|^vc_|^elementor/.test(actionVal)) continue;
-            } catch {
+          for (const lm of linkMappings) {
+            const safety = assessActionSafety(lm.label, domainProfile);
+            if (!safety.allowed) {
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                screenId: screen.id,
+                jobId: opts.jobId,
+                type: "ACTION_BLOCKED_BY_SAFETY_POLICY",
+                payload: { label: lm.label, href: lm.href, decision: safety, profile: domainProfile.industry }
+              }).catch(() => {
+              });
+              continue;
             }
-            const canonHref = canonicalize(href);
-            if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 2) {
+            pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
+            AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
+              domain: new URL(url).hostname,
+              purpose: `navigate:${lm.label}`,
+              fingerprint: sha256(`${canonicalize(url)}:${lm.label}:${canonicalize(lm.href)}`),
+              steps: [{ action: "click", label: lm.label, targetUrl: lm.href, fromUrl: url }],
+              successRate: 0.5
+            }).catch(() => {
+            });
+            if (!selective) {
               try {
-                const hrefPath = new URL(href).pathname;
-                if (isGeneratedDynamicPath(hrefPath)) continue;
+                const targetPath = new URL(lm.href).pathname;
+                const canonLmHref = canonicalize(lm.href);
+                if (!SKIP_PATH.test(targetPath) && isUsefulNavigationUrl(lm.href, origin) && !queued.has(canonLmHref) && !visited.has(canonLmHref) && queued.size < maxScreens * 2) {
+                  if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
+                  queued.add(canonLmHref);
+                  queue.push(lm.href, { parentElements: elements.length });
+                  edges.push({ fromUrl: url, toUrl: lm.href });
+                  logger.info({ fromUrl: url, newUrl: lm.href, label: lm.label, queueSize: queue.length }, "[crawler] anchor mapping added to queue");
+                }
               } catch {
               }
-              if (robotsCache && !await robotsCache.isAllowed(href)) continue;
-              queued.add(canonHref);
-              queue.push(href, { parentElements: elements.length });
-              logger.info({ newUrl: href, queueSize: queue.length }, "[crawler] new URL added to queue");
             }
           }
-          logger.info({ fromUrl: url, discovered: nextLinks.length, queueAfter: queue.length, visitedSoFar: visited.size }, "[crawler] link processing done");
-          try {
-            const jsRoutes = await page.evaluate(() => {
-              const routes = window.__zetaDiscoveredRoutes ?? [];
-              window.__zetaDiscoveredRoutes = [];
-              return routes;
-            }).catch(() => []);
-            for (const route of jsRoutes) {
+          if (!selective) {
+            for (const href of nextLinks) {
+              edges.push({ fromUrl: url, toUrl: href });
               try {
-                const abs = route.startsWith("http") ? route : `${origin}${route.startsWith("/") ? "" : "/"}${route}`;
-                const canonRoute = canonicalize(abs);
-                if (!queued.has(canonRoute) && !visited.has(canonRoute)) {
-                  queued.add(canonRoute);
-                  queue.push(abs);
-                  logger.info({ route: abs }, "[crawler] history.pushState: discovered SPA route");
-                }
+                const hrefParams = new URL(href).searchParams;
+                const actionVal = hrefParams.get("action") ?? "";
+                if (/^yith-|^wc-|^vc_|^elementor/.test(actionVal)) continue;
               } catch {
               }
+              const canonHref = canonicalize(href);
+              if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 2) {
+                try {
+                  const hrefPath = new URL(href).pathname;
+                  if (isGeneratedDynamicPath(hrefPath)) continue;
+                } catch {
+                }
+                if (robotsCache && !await robotsCache.isAllowed(href)) continue;
+                queued.add(canonHref);
+                queue.push(href, { parentElements: elements.length });
+                logger.info({ newUrl: href, queueSize: queue.length }, "[crawler] new URL added to queue");
+              }
             }
-          } catch {
-          }
-          if (preDomCount < 25 && !revisitQueue.includes(url)) {
+            logger.info({ fromUrl: url, discovered: nextLinks.length, queueAfter: queue.length, visitedSoFar: visited.size }, "[crawler] link processing done");
             try {
-              const postDomCount = await page.evaluate(
-                () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
-              ).catch(() => 0);
-              if (postDomCount > preDomCount * 1.3 + 4) {
-                revisitQueue.push(url);
-                logger.info({ url, preDomCount, postDomCount }, "[P3a] Async content detected \u2014 scheduled for re-visit");
+              const jsRoutes = await page.evaluate(() => {
+                const routes = window.__zetaDiscoveredRoutes ?? [];
+                window.__zetaDiscoveredRoutes = [];
+                return routes;
+              }).catch(() => []);
+              for (const route of jsRoutes) {
+                try {
+                  const abs = route.startsWith("http") ? route : `${origin}${route.startsWith("/") ? "" : "/"}${route}`;
+                  const canonRoute = canonicalize(abs);
+                  if (!queued.has(canonRoute) && !visited.has(canonRoute)) {
+                    queued.add(canonRoute);
+                    queue.push(abs);
+                    logger.info({ route: abs }, "[crawler] history.pushState: discovered SPA route");
+                  }
+                } catch {
+                }
               }
             } catch {
             }
-          }
-          if (visited.size <= 5) {
-            try {
-              const jsDetectedRoutes = await detectJsRoutes(page, origin);
-              for (const route of jsDetectedRoutes) {
-                const canonRoute = canonicalize(route);
-                if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 2) {
-                  queued.add(canonRoute);
-                  queue.push(route, { parentElements: elements.length });
-                  logger.info({ route }, "[P2] JS router: discovered route added to queue");
+            const asyncContentCheckMinDom = parseInt(process.env.CRAWLER_ASYNC_CONTENT_MIN_DOM ?? "") || 25;
+            if (preDomCount < asyncContentCheckMinDom && !revisitQueue.includes(url)) {
+              try {
+                const postDomCount = await page.evaluate(
+                  () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
+                ).catch(() => 0);
+                const asyncGrowthRatio = parseFloat(process.env.CRAWLER_ASYNC_GROWTH_RATIO ?? "") || 1.3;
+                const asyncGrowthFloor = parseInt(process.env.CRAWLER_ASYNC_GROWTH_FLOOR ?? "") || 4;
+                if (postDomCount > preDomCount * asyncGrowthRatio + asyncGrowthFloor) {
+                  revisitQueue.push(url);
+                  logger.info({ url, preDomCount, postDomCount }, "[P3a] Async content detected \u2014 scheduled for re-visit");
                 }
+              } catch {
               }
-            } catch {
             }
-          }
-          for (const relPath of [...xhrDiscoveredPaths]) {
-            xhrDiscoveredPaths.delete(relPath);
-            try {
-              const abs = `${origin}${relPath}`;
-              const canon = canonicalize(abs);
-              if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
-                queued.add(canon);
-                queue.push(abs);
-                logger.info({ url: abs }, "[P4] XHR: page URL from JSON response added to queue");
+            if (visited.size <= (parseInt(process.env.CRAWLER_SPA_ROUTER_DETECT_LIMIT ?? "") || 5)) {
+              try {
+                const jsDetectedRoutes = await detectJsRoutes(page, origin);
+                for (const route of jsDetectedRoutes) {
+                  const canonRoute = canonicalize(route);
+                  if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 2) {
+                    queued.add(canonRoute);
+                    queue.push(route, { parentElements: elements.length });
+                    logger.info({ route }, "[P2] JS router: discovered route added to queue");
+                  }
+                }
+              } catch {
               }
-            } catch {
             }
-          }
-          if (nextLinks.length === 0 && elements.length < 5 && opts.skyvernConfig) {
-            try {
-              logger.info({ url }, "Gap 5: thin page detected \u2014 asking Skyvern to discover navigation URLs");
-              const skyvernNav = await skyvernNavigate(
-                url,
-                ["Find and return all navigation links, menu items, and page URLs accessible from this screen"],
-                opts.skyvernConfig
-              );
-              for (const skyvernUrl of skyvernNav.visitedUrls) {
-                const canonSkyvernUrl = canonicalize(skyvernUrl);
-                if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 2) {
-                  logger.info({ url: skyvernUrl }, "Gap 5: Skyvern-discovered URL added to ZeTa crawl queue");
-                  queued.add(canonSkyvernUrl);
-                  queue.push(skyvernUrl);
-                  pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: "Skyvern-discovered link", targetUrl: skyvernUrl });
+            for (const relPath of [...xhrDiscoveredPaths]) {
+              xhrDiscoveredPaths.delete(relPath);
+              try {
+                const abs = `${origin}${relPath}`;
+                const canon = canonicalize(abs);
+                if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                  queued.add(canon);
+                  queue.push(abs);
+                  logger.info({ url: abs }, "[P4] XHR: page URL from JSON response added to queue");
                 }
+              } catch {
+              }
+            }
+            if (nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? "") || 5) && opts.skyvernConfig) {
+              try {
+                logger.info({ url }, "Gap 5: thin page detected \u2014 asking Skyvern to discover navigation URLs");
+                const skyvernNav = await skyvernNavigate(
+                  url,
+                  ["Find and return all navigation links, menu items, and page URLs accessible from this screen"],
+                  opts.skyvernConfig
+                );
+                for (const skyvernUrl of skyvernNav.visitedUrls) {
+                  const canonSkyvernUrl = canonicalize(skyvernUrl);
+                  if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 2) {
+                    logger.info({ url: skyvernUrl }, "Gap 5: Skyvern-discovered URL added to ZeTa crawl queue");
+                    queued.add(canonSkyvernUrl);
+                    queue.push(skyvernUrl);
+                    pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: "Skyvern-discovered link", targetUrl: skyvernUrl });
+                  }
+                }
+              } catch {
               }
-            } catch {
             }
           }
         }
@@ -2569,7 +3201,8 @@ ${snippet}`,
       };
     }
     const successfulScreenCount = discovered.filter((d) => d.screenId).length;
-    if (successfulScreenCount < 10 && opts.skyvernConfig) {
+    const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? "") || 10;
+    if (successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
       await onProgress?.(78, "Few screens found \u2014 using Skyvern to discover additional navigation URLs");
       const currentUrl = page.url();
       const skyvernNav = await skyvernNavigate(
@@ -2587,16 +3220,16 @@ ${snippet}`,
           if (visited.size >= maxScreens) break;
           try {
             await page.goto(skyvernUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
-            await page.waitForTimeout(1500);
+            await new Promise((r) => setTimeout(r, 1500));
             const skyvernName = new URL(skyvernUrl).pathname.split("/").filter(Boolean).pop() ?? "Screen";
             const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, skyvernName, skyvernUrl, opts.jobId, "SKYVERN_AI");
             const shotPath = `${opts.screenshotDir ?? "/tmp"}/${screen.id}-v${screen.version}.png`;
-            await waitForScreenshotReady(page);
+            await waitForScreenshotReady(page, rawPage ?? void 0);
             await page.screenshot({ path: shotPath, fullPage: true });
             await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP");
             const [elems, links, linkMappings] = await Promise.all([
-              extractElements(stagehand, page, elementLLM ?? void 0),
-              discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0),
+              extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0),
+              discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0),
               extractLinkMappings(page, origin)
             ]);
             for (const el of elems) {
@@ -2626,15 +3259,15 @@ ${snippet}`,
           if (!isUsefulNavigationUrl(href, origin)) continue;
           try {
             await page.goto(href, { waitUntil: "domcontentloaded", timeout: 2e4 });
-            await page.waitForTimeout(1e3);
+            await new Promise((r) => setTimeout(r, 1e3));
             const nm = new URL(href).pathname.split("/").filter(Boolean).pop() ?? "Screen";
             const sc = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, "SKYVERN_AI");
             const sp = `${opts.screenshotDir ?? "/tmp"}/${sc.id}-v${sc.version}.png`;
-            await waitForScreenshotReady(page);
+            await waitForScreenshotReady(page, rawPage ?? void 0);
             await page.screenshot({ path: sp, fullPage: true });
             await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, "DESKTOP");
             const [elems2, linkMappings2] = await Promise.all([
-              extractElements(stagehand, page, elementLLM ?? void 0),
+              extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0),
               extractLinkMappings(page, origin)
             ]);
             for (const el of elems2) {
@@ -2680,28 +3313,51 @@ ${snippet}`,
         );
       }
     } catch (err) {
-      logger.warn({ err: String(err) }, "[P1] URL pattern completion failed \u2014 non-fatal, continuing");
+      const msg = String(err);
+      logger.warn({ err: msg }, "[P1] URL pattern completion failed \u2014 non-fatal, continuing");
+      crawlWarnings.push(`url-completer skipped: ${msg.includes("401") || msg.toLowerCase().includes("api key") || msg.toLowerCase().includes("authentication") ? "LLM API key invalid or missing" : msg}`);
     }
     if (revisitQueue.length > 0) {
       await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content\u2026`);
       logger.info({ count: revisitQueue.length }, "[P3a] Starting async content re-visit pass");
       for (const revisitUrl of revisitQueue.slice(0, 10)) {
-        const existingScreenId = visited.get(canonicalize(revisitUrl));
-        if (!existingScreenId) continue;
+        const rawScreenId = visited.get(canonicalize(revisitUrl));
+        const isAuthWallRevisit = rawScreenId === "_auth_wall_";
+        if (!rawScreenId) continue;
         try {
           await page.goto(revisitUrl, { waitUntil: "networkidle", timeout: 25e3 });
-          await page.waitForTimeout(2e3);
+          await new Promise((r) => setTimeout(r, 2e3));
+          await waitForScreenshotReady(page, rawPage ?? void 0);
+          const landedUrl = (rawPage ?? page).url?.() ?? revisitUrl;
+          const landedCanon = canonicalize(landedUrl);
+          let existingScreenId = isAuthWallRevisit ? visited.get(landedCanon) ?? null : rawScreenId;
+          if (isAuthWallRevisit && (!existingScreenId || existingScreenId.startsWith("_")) && !visited.has(landedCanon)) {
+            const pathParts = new URL(landedUrl).pathname.split("/").filter(Boolean);
+            const screenName = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
+            const newScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, landedUrl, opts.jobId, "LINK_FOLLOW").catch(() => null);
+            if (newScreen) {
+              visited.set(landedCanon, newScreen.id);
+              existingScreenId = newScreen.id;
+            }
+          }
+          if (!existingScreenId || existingScreenId.startsWith("_")) {
+            logger.warn({ url: revisitUrl, landed: landedUrl }, "[P3a] Re-visit skipped \u2014 could not resolve screen ID");
+            continue;
+          }
           const shotPath2 = `${opts.screenshotDir ?? "/tmp"}/${existingScreenId}-revisit.png`;
-          await waitForScreenshotReady(page);
           await page.screenshot({ path: shotPath2, fullPage: true });
-          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, "DESKTOP").catch(() => {
+          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, "DESKTOP", void 0).catch(() => {
           });
-          const revisitElements = await extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []);
-          for (const el of revisitElements) {
-            await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+          const revisitElements = await extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []);
+          if (revisitElements.length > 0) {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, existingScreenId).catch(() => {
             });
+            for (const el of revisitElements) {
+              await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+              });
+            }
           }
-          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false).catch(() => []);
+          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []);
           for (const href of revisitLinks) {
             const canon = canonicalize(href);
             if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
@@ -2709,13 +3365,68 @@ ${snippet}`,
               queue.push(href);
             }
           }
-          logger.info({ url: revisitUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, "[P3a] Re-visit complete");
+          logger.info({ url: revisitUrl, landed: landedUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, "[P3a] Re-visit complete");
         } catch (err) {
           logger.warn({ url: revisitUrl, err: String(err) }, "[P3a] Re-visit failed \u2014 non-fatal");
         }
       }
     }
-    if (visited.size >= 3) {
+    if (queue.length > 0 && visited.size < maxScreens) {
+      const p3bCount = Math.min(queue.length, maxScreens - visited.size);
+      logger.info({ newLinks: p3bCount }, "[P3b] Extended BFS \u2014 processing links discovered during revisit pass");
+      await onProgress?.(82, `Extended crawl: ${p3bCount} newly discovered page(s)\u2026`);
+      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        const href = queue.shift();
+        if (!href) break;
+        const canonHref = canonicalize(href);
+        if (visited.has(canonHref)) continue;
+        if (!isUsefulNavigationUrl(href, origin)) continue;
+        visited.set(canonHref, "_pending_");
+        try {
+          await page.goto(href, { waitUntil: "networkidle", timeout: 25e3 }).catch(() => {
+          });
+          await waitForScreenshotReady(page, rawPage ?? void 0);
+          const pathParts = new URL(href).pathname.split("/").filter(Boolean);
+          const nm = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, "LINK_FOLLOW");
+          visited.set(canonHref, screen.id);
+          const sp = `${opts.screenshotDir ?? "/tmp"}/${screen.id}-v${screen.version}.png`;
+          await page.screenshot({ path: sp, fullPage: true }).catch(() => {
+          });
+          const cloudUrl3 = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => "");
+          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, sp, screen.version, "DESKTOP", cloudUrl3).catch(() => {
+          });
+          const [p3bElems, p3bLinks] = await Promise.all([
+            extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []),
+            discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).catch(() => [])
+          ]);
+          if (p3bElems.length > 0) {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+            });
+            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 4) {
+              const _b3 = p3bElems.slice(_ei3, _ei3 + 4);
+              await Promise.all(_b3.map(
+                (el) => AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
+                })
+              ));
+            }
+          }
+          for (const link of p3bLinks) {
+            const c = canonicalize(link);
+            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 2) {
+              queued.add(c);
+              queue.push(link);
+            }
+          }
+          discovered.push({ url: href, screenId: screen.id, elements: p3bElems.length, version: screen.version });
+          logger.info({ url: href, elements: p3bElems.length, newLinks: p3bLinks.length }, "[P3b] Extended BFS screen captured");
+        } catch (err) {
+          logger.warn({ url: href, err: String(err) }, "[P3b] Extended BFS URL failed");
+          visited.set(canonHref, "_error_");
+        }
+      }
+    }
+    if (visited.size >= (parseInt(process.env.CRAWLER_GAP_PASS_MIN_SCREENS ?? "") || 3)) {
       try {
         await onProgress?.(80, "Running gap analysis\u2026");
         const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
@@ -2739,7 +3450,7 @@ ${snippet}`,
           });
           logger.info({ confirmed: confirmedGaps.length, total: gapCandidates.length }, "[P6] Gap pass: visiting confirmed gap URLs");
           const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
-          for (const gap of confirmedGaps.slice(0, 8)) {
+          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? "") || 8)) {
             const gapUrl = gap.confirmedUrl;
             if (visited.size >= maxScreens) break;
             try {
@@ -2748,16 +3459,16 @@ ${snippet}`,
                 logger.warn({ gapUrl, landed: page.url() }, "[P6] Gap URL redirected to login \u2014 skipping");
                 continue;
               }
-              await page.waitForTimeout(1500);
+              await new Promise((r) => setTimeout(r, 1500));
               const gapName = gap.category;
               const gapScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, gapName, gapUrl, opts.jobId, "ENTRY_POINT");
               const gapShot = `${opts.screenshotDir ?? "/tmp"}/${gapScreen.id}-v${gapScreen.version}.png`;
-              await waitForScreenshotReady(page);
+              await waitForScreenshotReady(page, rawPage ?? void 0);
               await page.screenshot({ path: gapShot, fullPage: true });
               await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, "DESKTOP");
               const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
-                extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []),
-                discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false).catch(() => []),
+                extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []),
+                discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []),
                 extractLinkMappings(page, origin).catch(() => [])
               ]);
               for (const el of gapElems) {
@@ -2785,11 +3496,14 @@ ${snippet}`,
           logger.info({ total: gapCandidates.length }, "[P6] Gap pass: no confirmed gaps found");
         }
       } catch (err) {
-        logger.warn({ err: String(err) }, "[P6] Gap pass failed \u2014 non-fatal, continuing");
+        const msg = String(err);
+        logger.warn({ err: msg }, "[P6] Gap pass failed \u2014 non-fatal, continuing");
+        crawlWarnings.push(`gap-analysis skipped: ${msg.includes("401") || msg.toLowerCase().includes("api key") || msg.toLowerCase().includes("authentication") ? "LLM API key invalid or missing" : msg}`);
       }
     }
     if (!selective && visited.size > 0) {
-      const discoveredUrls = Array.from(visited.keys());
+      const VISIT_MARKER_RE = /^_|^pre-auth$|^skipped$/;
+      const discoveredUrls = Array.from(visited.entries()).filter(([, v]) => typeof v === "string" && !VISIT_MARKER_RE.test(v)).map(([k]) => k);
       await AppBrain.markRemovedScreens(opts.tenantId, opts.projectId, discoveredUrls, opts.jobId);
     }
     await onProgress?.(82, "Saving app map");
@@ -2964,6 +3678,10 @@ ${snippet}`,
             ignoreHTTPSErrors: true,
             ...mainStorageState ? { storageState: mainStorageState } : {}
           });
+          mobileCtx.on("download", (dl) => {
+            dl.cancel().catch(() => {
+            });
+          });
           const mobilePage = await mobileCtx.newPage();
           for (const disc of discovered) {
             if (!disc.screenId) continue;
@@ -2976,11 +3694,13 @@ ${snippet}`,
                 await mobilePage.addStyleTag({ content: "*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }" });
               } catch {
               }
+              await infiniteScrollToBottom(mobilePage, { maxScrolls: 20, scrollDelay: 300, stabilizeMs: 400 }, mobilePage).catch(() => {
+              });
               const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${disc.screenId}-${profile.toLowerCase()}-v${disc.version}.png`);
-              await waitForScreenshotReady(mobilePage);
+              await waitForScreenshotReady(mobilePage, mobilePage);
               const [, vpElements] = await Promise.all([
                 mobilePage.screenshot({ path: shotPath, fullPage: true }),
-                domFallbackElements(mobilePage)
+                domFallbackElements(mobilePage, mobilePage)
               ]);
               const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
               await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version, profile, cloudUrl);
@@ -3033,6 +3753,22 @@ ${snippet}`,
       const probeInputs = discovered.filter((d) => d.screenId).map((d) => ({ url: d.url, screenId: d.screenId, elements: d.elementFacts ?? [] }));
       await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
     }
+    try {
+      await onProgress?.(90, "Generating state graph coverage report");
+      const stateGraph = new ScreenTransitionGraph();
+      const coveragePayload = {
+        stateGraph: stateGraph.getCoverageReport(),
+        totalScreensCrawled: discovered.filter((d) => !!d.screenId).length
+      };
+      AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+        jobId: opts.jobId,
+        type: "CRAWL_COVERAGE_REPORT",
+        payload: coveragePayload
+      }).catch(() => {
+      });
+    } catch (extErr) {
+      logger.warn({ err: String(extErr) }, "[coverage-report] non-fatal error");
+    }
     await onProgress?.(97, "Running post-crawl analysis");
     const successScreens = discovered.filter((d) => !!d.screenId);
     ;
@@ -3113,6 +3849,14 @@ ${snippet}`,
     const hasMoreSegments = pendingUrls.length > 0;
     const screensSucceeded = discovered.filter((d) => !!d.screenId).length;
     const screensErrored = discovered.filter((d) => !!d.error && !d.screenId).length;
+    if (screensSucceeded >= 3) {
+      const screensWithElements = discovered.filter((d) => (d.elements ?? 0) > 0 && d.screenId).length;
+      if (screensWithElements === 0) {
+        crawlWarnings.push(
+          `element-extraction: 0 elements found across all ${screensSucceeded} screens \u2014 DOM evaluate may be failing (check server logs for "page.evaluate threw" warnings)`
+        );
+      }
+    }
     return {
       screensFound: screensSucceeded,
       // successful captures only (not errors)
@@ -3129,8 +3873,9 @@ ${snippet}`,
       segmentIndex: segIdx,
       // which segment this was
       hasMoreSegments,
-      domHashes: Array.from(seenDomHashes)
+      domHashes: Array.from(seenDomHashes),
       // carry forward to next segment to avoid re-screenshotting
+      crawlWarnings: crawlWarnings.length > 0 ? crawlWarnings : void 0
     };
   } finally {
     if (stagehandPromptTokens > 0 || stagehandCompletionTokens > 0) {
@@ -3165,8 +3910,8 @@ ${snippet}`,
   }
 }
 const ELEMENT_NOISE = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
-async function extractElements(stagehand, page, llmCfg) {
-  const domElements = await domFallbackElements(page).catch((err) => {
+async function extractElements(stagehand, page, llmCfg, rawPage) {
+  const domElements = await domFallbackElements(page, rawPage).catch((err) => {
     console.warn("[crawler] domFallbackElements failed (page context issue or DOM error):", err?.message ?? err);
     return [];
   });
@@ -3204,7 +3949,10 @@ async function extractElements(stagehand, page, llmCfg) {
       dynamicElements = [];
     }
   }
-  const contentElements = await contentFallbackElements(page);
+  const contentElements = await contentFallbackElements(page).catch((err) => {
+    console.warn("[crawler] contentFallbackElements failed (page context issue):", err?.message ?? err);
+    return [];
+  });
   const crossSourceDedup = (arr) => {
     const stripPhrases = (s) => s.replace(/^button to expand ['"]?|['"]?\s*details$/gi, "").replace(/\s+(faq\s*)?button$/i, "").replace(/^button:\s*/i, "").toLowerCase().trim().slice(0, 80);
     const ROLE_PRIORITY = { "faq-question": 10, "faq-answer": 9, "h1": 8, "h2": 7, "h3": 6, "h4": 5, "heading": 8, button: 3, a: 2 };
@@ -3245,9 +3993,9 @@ async function extractElements(stagehand, page, llmCfg) {
     return domElements;
   }
 }
-async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false) {
+async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false, rawPage) {
   const navExpanded = await expandNavMenus(page);
-  const domLinks = await domFallbackLinks(page, origin);
+  const domLinks = await domFallbackLinks(page, origin, rawPage);
   const interactiveLinks = await interactAndCollectLinks(page, origin);
   if (navExpanded) {
     page.keyboard?.press("Escape").catch(() => {
@@ -3294,8 +4042,8 @@ async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false) {
     return allDomLinks;
   }
 }
-async function domFallbackElements(page) {
-  return page.evaluate(() => {
+async function domFallbackElements(page, rawPage) {
+  function _browserEval() {
     const out = [];
     const seen = /* @__PURE__ */ new Set();
     const noise = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
@@ -3311,7 +4059,7 @@ async function domFallbackElements(page) {
       }
       return results;
     };
-    const selector = 'input:not([type=hidden]),select,textarea,button,[role="button"],[role="checkbox"],[role="radio"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],[role="gridcell"],[role="link"],a[role="button"],[contenteditable="true"]';
+    const selector = 'input:not([type=hidden]),select,textarea,button,a[href],[role="button"],[role="checkbox"],[role="radio"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],[role="gridcell"],[role="link"],[contenteditable="true"]';
     querySelectorAllDeep(document, selector).forEach((el) => {
       const style = window.getComputedStyle(el);
       if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0" || style.pointerEvents === "none") return;
@@ -3323,11 +4071,10 @@ async function domFallbackElements(page) {
       const tag = el.tagName.toLowerCase();
       const ariaRole = el.getAttribute("role");
       const isActionEl = tag === "button" || tag === "a" || ariaRole === "button";
-      if (tag === "a" || ariaRole === "button" && tag !== "button") {
+      if (tag === "a") {
         let p = el.parentElement;
         while (p) {
-          const ptag = p.tagName?.toLowerCase();
-          if (["header", "footer"].includes(ptag)) return;
+          if (p.tagName?.toLowerCase() === "footer") return;
           p = p.parentElement;
         }
       }
@@ -3472,7 +4219,48 @@ async function domFallbackElements(page) {
     const priority = (e) => ["email", "password", "text", "tel", "date", "number", "select", "textarea", "url", "file"].includes(e.role) ? 0 : 1;
     out.sort((a, b) => priority(a) - priority(b));
     return out;
-  }).then((elems) => elems.map((e) => ({ ...e, isInteractive: true }))).catch(() => []);
+  }
+  const _browserEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_browserEval.toString()})()})();`;
+  if (rawPage && typeof rawPage.context === "function") {
+    try {
+      const ctx = rawPage.context();
+      if (typeof ctx?.newCDPSession === "function") {
+        const cdp = await ctx.newCDPSession(rawPage);
+        try {
+          const evalResult = await cdp.send("Runtime.evaluate", {
+            expression: _browserEvalStr,
+            returnByValue: true,
+            awaitPromise: false
+          });
+          if (evalResult?.exceptionDetails) {
+            console.warn("[crawler] domFallbackElements CDP primary error:", evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
+          } else if (Array.isArray(evalResult?.result?.value)) {
+            return evalResult.result.value.map((e) => ({ ...e, isInteractive: true }));
+          }
+        } finally {
+          cdp.detach().catch(() => {
+          });
+        }
+      }
+    } catch (cdpPrimaryErr) {
+      console.warn("[crawler] domFallbackElements CDP primary failed, trying page.evaluate:", cdpPrimaryErr?.message ?? String(cdpPrimaryErr));
+    }
+  }
+  const evalPage = rawPage ?? page;
+  const _doEvaluate = () => evalPage.evaluate(_browserEvalStr);
+  let result;
+  try {
+    result = await _doEvaluate();
+  } catch (err1) {
+    await new Promise((r) => setTimeout(r, 500));
+    try {
+      result = await _doEvaluate();
+    } catch (err2) {
+      console.warn("[crawler] domFallbackElements page.evaluate threw (attempt 2):", err2?.message ?? String(err2));
+      return [];
+    }
+  }
+  return (result ?? []).map((e) => ({ ...e, isInteractive: true }));
 }
 async function contentFallbackElements(page) {
   return page.evaluate(() => {
@@ -4272,8 +5060,8 @@ async function expandNavMenus(page) {
     return false;
   }
 }
-async function domFallbackLinks(page, origin) {
-  return page.evaluate((origin2) => {
+async function domFallbackLinks(page, origin, rawPage) {
+  function _linkEval(originStr) {
     const logout = /logout|log out|sign out|delete/i;
     const isRouteHash = (hash) => /^#!?\//.test(hash);
     const seen = /* @__PURE__ */ new Set();
@@ -4285,7 +5073,7 @@ async function domFallbackLinks(page, origin) {
       try {
         const u = new URL(rawHref, location.href);
         if (!isRouteHash(u.hash)) u.hash = "";
-        if (new URL(u).origin === origin2) seen.add(u.toString());
+        if (new URL(u).origin === originStr) seen.add(u.toString());
       } catch {
       }
     };
@@ -4316,7 +5104,39 @@ async function domFallbackLinks(page, origin) {
       }
     }
     return Array.from(seen);
-  }, origin).catch(() => []);
+  }
+  const _linkEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_linkEval.toString()})(${JSON.stringify(origin)})})();`;
+  if (rawPage && typeof rawPage.context === "function") {
+    try {
+      const ctx = rawPage.context();
+      if (typeof ctx?.newCDPSession === "function") {
+        const cdp = await ctx.newCDPSession(rawPage);
+        try {
+          const evalResult = await cdp.send("Runtime.evaluate", {
+            expression: _linkEvalStr,
+            returnByValue: true,
+            awaitPromise: false
+          });
+          if (!evalResult?.exceptionDetails && Array.isArray(evalResult?.result?.value)) {
+            return evalResult.result.value;
+          }
+          if (evalResult?.exceptionDetails) {
+            console.warn("[crawler] domFallbackLinks CDP error:", evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
+          }
+        } finally {
+          cdp.detach().catch(() => {
+          });
+        }
+      }
+    } catch (cdpErr) {
+      console.warn("[crawler] domFallbackLinks CDP failed:", cdpErr?.message ?? String(cdpErr));
+    }
+  }
+  const evalPage = rawPage ?? page;
+  return evalPage.evaluate(_linkEvalStr).catch((err) => {
+    console.warn("[crawler] domFallbackLinks page.evaluate threw:", err?.message ?? String(err));
+    return [];
+  });
 }
 async function extractLinkMappings(page, origin) {
   return page.evaluate((origin2) => {
@@ -4417,50 +5237,105 @@ async function dismissOverlays(page) {
     } catch {
     }
   }
-  await page.waitForTimeout(300).catch(() => {
-  });
+  await new Promise((r) => setTimeout(r, 300));
 }
-async function scrollToTriggerLazyLoad(page) {
+async function scrollToTriggerLazyLoad(page, rawPage) {
+  const evalPage = rawPage ?? page;
   try {
-    await page.evaluate(async () => {
-      const step = Math.max(window.innerHeight, 400);
-      const total = Math.min(document.body.scrollHeight, 12e3);
-      for (let y = step; y < total; y += step) {
-        window.scrollTo(0, y);
-        await new Promise((r) => setTimeout(r, 120));
+    const { scrollHeight, innerHeight, innerWidth } = await evalPage.evaluate(() => ({
+      scrollHeight: Math.min(document.body.scrollHeight, 12e3),
+      innerHeight: window.innerHeight || 800,
+      innerWidth: window.innerWidth || 1280
+    })).catch(() => ({ scrollHeight: 0, innerHeight: 800, innerWidth: 1280 }));
+    if (scrollHeight > innerHeight) {
+      let cdp = null;
+      if (rawPage && typeof rawPage.context === "function") {
+        try {
+          const ctx = rawPage.context();
+          if (typeof ctx?.newCDPSession === "function") cdp = await ctx.newCDPSession(rawPage);
+        } catch {
+          cdp = null;
+        }
       }
-      window.scrollTo(0, 0);
-    });
-    await page.waitForTimeout(400).catch(() => {
-    });
-    await waitForStableDOM(page, 3e3, 300);
+      const step = Math.max(innerHeight, 400);
+      for (let y = step; y < scrollHeight; y += step) {
+        await evalPage.evaluate((pos) => window.scrollTo(0, pos), y).catch(() => {
+        });
+        if (cdp) {
+          await cdp.send("Input.dispatchMouseEvent", {
+            type: "mouseWheel",
+            x: Math.floor(innerWidth / 2),
+            y: Math.floor(innerHeight / 2),
+            deltaX: 0,
+            deltaY: step,
+            modifiers: 0
+          }).catch(() => {
+          });
+        }
+        await new Promise((r) => setTimeout(r, 150));
+      }
+      await evalPage.evaluate(() => window.scrollTo(0, 0)).catch(() => {
+      });
+      if (cdp) cdp.detach().catch(() => {
+      });
+    }
+    await new Promise((r) => setTimeout(r, 400));
+    const scrollWaitPage = rawPage ?? page;
+    if (typeof scrollWaitPage.waitForLoadState === "function") {
+      await scrollWaitPage.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
+      });
+    }
+    await waitForStableDOM(page, 4e3, 500, rawPage);
   } catch {
   }
 }
-async function waitForStableDOM(page, maxWait = 8e3, stabilityMs = 400) {
+async function waitForStableDOM(page, maxWait = 8e3, stabilityMs = 400, rawPage) {
+  const evalPage = rawPage ?? page;
   const deadline = Date.now() + maxWait;
   let lastCount = -1;
   let stableAt = 0;
+  let evalErrors = 0;
   while (Date.now() < deadline) {
-    const count = await page.evaluate(() => document.querySelectorAll("*").length).catch(() => 0);
-    if (count !== lastCount) {
-      lastCount = count;
-      stableAt = Date.now();
-    } else if (Date.now() - stableAt >= stabilityMs) {
-      return;
+    const count = await evalPage.evaluate(() => {
+      let c = 0;
+      const visit = (root) => {
+        const els = root.querySelectorAll("*");
+        c += els.length;
+        els.forEach((el) => {
+          if (el.shadowRoot) visit(el.shadowRoot);
+        });
+      };
+      visit(document);
+      return c;
+    }).catch(() => -1);
+    if (count === -1) {
+      if (++evalErrors >= 3) {
+        await new Promise((r) => setTimeout(r, stabilityMs));
+        return;
+      }
+    } else {
+      evalErrors = 0;
+      if (count !== lastCount) {
+        lastCount = count;
+        stableAt = Date.now();
+      } else if (Date.now() - stableAt >= stabilityMs) {
+        return;
+      }
     }
     await new Promise((r) => setTimeout(r, 100));
   }
 }
-async function waitForScreenshotReady(page) {
-  const hardDeadline = Date.now() + 2e4;
+async function waitForScreenshotReady(page, rawPage) {
+  const hardDeadline = Date.now() + 3e4;
   const withHardCap = (p) => Promise.race([p, new Promise((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
+  const waitPage = rawPage ?? page;
+  const evalPage = rawPage ?? page;
   try {
-    await withHardCap(page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => {
+    await withHardCap(waitPage.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => {
     }));
-    await withHardCap(page.waitForLoadState("networkidle", { timeout: 12e3 }).catch(() => {
+    await withHardCap(waitPage.waitForLoadState("networkidle", { timeout: 12e3 }).catch(() => {
     }));
-    await withHardCap(page.evaluate(() => {
+    await withHardCap(evalPage.evaluate(() => {
       if (document.readyState === "complete") return true;
       return new Promise((resolve) => {
         const done = () => resolve(true);
@@ -4469,14 +5344,14 @@ async function waitForScreenshotReady(page) {
       });
     }).catch(() => {
     }));
-    await withHardCap(page.evaluate(async () => {
+    await withHardCap(evalPage.evaluate(async () => {
       const fonts = document.fonts;
       if (fonts?.ready) await fonts.ready;
     }).catch(() => {
     }));
-    await withHardCap(scrollToTriggerLazyLoad(page));
-    await withHardCap(waitForStableDOM(page, 4e3, 700));
-    await withHardCap(page.evaluate(async () => {
+    await withHardCap(scrollToTriggerLazyLoad(rawPage ?? page, rawPage));
+    await withHardCap(waitForStableDOM(page, 4e3, 700, rawPage));
+    await withHardCap(evalPage.evaluate(async () => {
       const pending = Array.from(document.images).filter((img) => !img.complete);
       if (pending.length === 0) return;
       await Promise.race([
@@ -4488,7 +5363,7 @@ async function waitForScreenshotReady(page) {
       ]);
     }).catch(() => {
     }));
-    await withHardCap(waitForStableDOM(page, 2e3, 500));
+    await withHardCap(waitForStableDOM(page, 2e3, 500, rawPage));
   } catch {
   }
 }
@@ -4642,6 +5517,7 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
   return { captured: true, finalUrl, flowType };
 }
 async function runAuthDiscoveryPhase(page, opts, origin, onProgress) {
+  const tuning = await getCrawlTuning();
   const queue = [];
   const queued = /* @__PURE__ */ new Set();
   const capturedKeys = /* @__PURE__ */ new Set();
@@ -4677,8 +5553,11 @@ async function runAuthDiscoveryPhase(page, opts, origin, onProgress) {
   await onProgress?.(10, "Discovering public and auth flows");
   while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
     const current = queue.shift();
+    const _authPct = 10 + Math.round(captured / AUTH_DISCOVERY_MAX_SCREENS * 7);
+    const _authPath = current.url.replace(/^https?:\/\/[^/]+/, "") || "/";
+    await onProgress?.(_authPct, `Auth discovery \xB7 ${_authPath} \xB7 ${captured} found`);
     try {
-      const response = await page.goto(current.url, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => null);
+      const response = await page.goto(current.url, { waitUntil: "domcontentloaded", timeout: tuning.navTimeoutMs }).catch(() => null);
       if (response && response.status && response.status() >= 400) continue;
       await page.waitForLoadState("networkidle", { timeout: 6e3 }).catch(() => {
       });