@detiq/crawler 1.1.14 → 1.1.16

package/src/crawler.ts

@@ -74,6 +74,17 @@ import { screensToSitemapUrls, writeSitemap } from './sitemap-writer.js';
 import { generateOpenApiSpec, specToYaml } from './openapi-generator.js';
 import { WebhookNotifier } from './webhook-notifier.js';
 import { generatePlaywrightTest } from './playwright-recorder.js';
+import { enumerateDropdowns } from './dropdown-enumerator.js';
+import { probeCrudFlows } from './crud-interaction.js';
+import { crawlFiltersAndPagination } from './filter-pagination-crawler.js';
+import { probeMultiContext } from './multi-context-handler.js';
+import { probeAdvancedUI } from './advanced-ui-probes.js';
+import { ScreenTransitionGraph } from './state-graph.js';
+import { crawlNestedContexts } from './nested-window-crawler.js';
+import { crawlShadowDom } from './shadow-dom-crawler.js';
+import { crawlAccessibilityTree } from './accessibility-tree-crawler.js';
+import { collectPerfMetrics } from './performance-metrics.js';
+import { inspectServiceWorker } from './service-worker-interceptor.js';
 
 async function getCrawlTuning() {
   const { getSystemConfig } = await import('@detiq/core');
@@ -141,12 +152,27 @@ function isGeneratedDynamicPath(pathname: string): boolean {
   return normalized.split('/').filter(Boolean).some(isGeneratedPathSegment);
 }
 
+// File extensions that trigger browser downloads rather than page renders.
+// URLs ending with these are skipped from the BFS queue entirely.
+const DOWNLOAD_EXT_RE = /\.(pdf|zip|gz|tar|rar|7z|exe|dmg|pkg|deb|rpm|msi|apk|ipa|doc|docx|xls|xlsx|ppt|pptx|csv|json|xml|yaml|yml|txt|log|sql|db|sqlite|mp4|mp3|avi|mov|mkv|webm|wav|ogg|flac|jpg|jpeg|png|gif|bmp|svg|webp|ico|woff|woff2|ttf|eot|otf|bin|iso|img|vhd|vmdk)(\?.*)?$/i;
+
+// API endpoint path patterns — these return data (JSON/CSV/binary), not rendered pages.
+// Crawling them triggers file downloads and wastes BFS slots.
+const API_PATH_RE = /^\/api\//i;
+
+// Export/download path segments that appear in apps — these return file data even without
+// a file extension in the URL (e.g. /admin/exports/report, /download/invoice/123).
+const EXPORT_PATH_RE = /\/(exports?|downloads?|export-csv|export-pdf|export-excel|download-report|generate-report|report-download)(\/|$|\?)/i;
+
 function isUsefulNavigationUrl(href: string, siteOrigin: string): boolean {
   try {
     const u = new URL(href);
     if (!sameSite(u.toString(), siteOrigin)) return false;
     if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
     if (isGeneratedDynamicPath(u.pathname)) return false;
+    if (DOWNLOAD_EXT_RE.test(u.pathname)) return false;
+    if (API_PATH_RE.test(u.pathname)) return false;
+    if (EXPORT_PATH_RE.test(u.pathname)) return false;
     return true;
   } catch {
     return false;
@@ -695,7 +721,9 @@ export async function crawlProject(opts: {
     payload: { profile: domainProfile, routeDecision },
   }).catch((err: any) => console.warn('[crawler] profile evidence save failed:', err?.message ?? err));
 
-  if (!stagehandConfigured) {
+  if (!stagehandConfigured && !opts.discoveryMode) {
+    // Discovery mode never calls Stagehand AI — DOM link extraction only.
+    // Skip this guard so URL discovery works even without an AI provider configured.
     throw new Error(
       configuredProvider
         ? `No API key configured for provider "${configuredProvider}". Go to Workspace → Settings → AI Provider.`
@@ -1021,14 +1049,81 @@ export async function crawlProject(opts: {
     if (pwContext) {
       // P4: pass XHR URL discovery callback — adds relative paths to xhrDiscoveredPaths
       attachNetworkObserver(pwContext, capturedApiEndpoints, (relPath) => xhrDiscoveredPaths.add(relPath));
-      // OAuth/SSO popup handler — attach before login so we don't miss it
+      // Popup/new-window handler — fires for every window.open() call.
+      // Handles two cases:
+      //  1. OAuth/SSO provider windows — wait for close, then continue login flow.
+      //  2. App-generated popups (help, report, lightbox-in-window) — screenshot + save as screen.
+      const OAUTH_PROVIDER_RE = /google\.|github\.|microsoft\.|facebook\.|twitter\.|linkedin\.|okta\.|auth0\.|ping\.|onelogin\.|sso\.|oauth\.|\/authorize\?|\/oauth2\//i;
       pwContext.on('page', (popup: any) => {
-        const popupUrl = popup.url();
-        logger.info({ url: popupUrl }, '[crawler] OAuth popup detected — waiting for close');
-        popup.waitForEvent('close', { timeout: 120_000 })
-          .then(() => logger.info('[crawler] OAuth popup closed — SSO flow complete'))
-          .catch(() => logger.warn('[crawler] OAuth popup timed out or errored'));
+        // Wait for the popup to navigate to a real URL (about:blank initially)
+        popup.waitForURL((u: string) => !u.startsWith('about:'), { timeout: 10_000 })
+          .then(async () => {
+            const popupUrl = popup.url();
+            if (OAUTH_PROVIDER_RE.test(popupUrl)) {
+              // OAuth flow — just wait for it to close
+              logger.info({ url: popupUrl }, '[crawler] OAuth popup navigated to provider');
+              popup.waitForEvent('close', { timeout: 120_000 })
+                .then(() => logger.info('[crawler] OAuth popup closed — SSO flow complete'))
+                .catch(() => logger.warn('[crawler] OAuth popup timed out or errored'));
+              return;
+            }
+            // App popup — capture as a screen (skip API/export/download URLs)
+            try {
+              const _popupPath = (() => { try { return new URL(popupUrl).pathname; } catch { return ''; } })();
+              if (API_PATH_RE.test(_popupPath) || EXPORT_PATH_RE.test(_popupPath) || DOWNLOAD_EXT_RE.test(_popupPath)) return;
+            } catch {}
+            logger.info({ url: popupUrl }, '[crawler] App popup detected — capturing');
+            try {
+              await popup.waitForLoadState('domcontentloaded', { timeout: 10_000 }).catch(() => {});
+              await popup.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => {});
+              const popupTitle = await popup.title().catch(() => '');
+              const popupName = popupTitle || (() => { try { return new URL(popupUrl).pathname.split('/').filter(Boolean).pop() || 'Popup'; } catch { return 'Popup'; } })();
+              const popupHash = popupUrl.replace(/[^a-z0-9]/gi, '').slice(-12);
+              const popupShotPath = path.join(opts.screenshotDir ?? '/tmp', `popup-${popupHash}.png`);
+              await waitForScreenshotReady(popup, popup);
+              await popup.screenshot({ path: popupShotPath, fullPage: false }).catch(() => {});
+              const popupScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, popupName, popupUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
+              if (popupScreen) {
+                const cloudUrl = await uploadToCloud(popupShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(popupShotPath)}`).catch(() => undefined);
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, popupScreen.id, popupShotPath, popupScreen.version, 'DESKTOP', cloudUrl).catch(() => {});
+                const popupEls = await domFallbackElements(popup, popup).catch(() => []);
+                for (const el of popupEls) {
+                  AppBrain.saveElement(opts.tenantId, opts.projectId, popupScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {});
+                }
+                logger.info({ url: popupUrl, name: popupName, elementCount: popupEls.length }, '[crawler] App popup screen saved');
+              }
+            } catch (popupErr: any) {
+              logger.warn({ url: popupUrl, err: popupErr?.message }, '[crawler] App popup capture failed — non-fatal');
+            }
+          })
+          .catch(() => {}); // popup may close before navigating (race condition)
       });
+      // Cancel any file downloads triggered by link navigation — the crawler captures
+      // pages, not binary files. Without this, clicking download links opens the browser
+      // download UI and stalls the crawl waiting for a page load that never comes.
+      pwContext.on('download', (dl: any) => { dl.cancel().catch(() => {}); });
+
+      // Route-level interception: immediately abort navigation to known API/export/download
+      // paths that would trigger file downloads. Uses URL-only matching (no double-fetch)
+      // to avoid the extra network round-trip that was causing "Failed to fetch" crashes.
+      // The download event handler above catches any remaining downloads that slip through.
+      try {
+        await pwContext.route('**/*', async (route: any, request: any) => {
+          const resourceType: string = request.resourceType?.() ?? '';
+          if (resourceType !== 'document') { await route.continue().catch(() => {}); return; }
+          try {
+            const u = new URL(request.url());
+            if (API_PATH_RE.test(u.pathname) || EXPORT_PATH_RE.test(u.pathname) || DOWNLOAD_EXT_RE.test(u.pathname)) {
+              logger.info({ url: request.url() }, '[crawler] route: aborting API/export/download navigation');
+              await route.abort('failed').catch(() => {});
+              return;
+            }
+          } catch {}
+          await route.continue().catch(() => {});
+        });
+      } catch (routeErr: any) {
+        logger.warn({ err: routeErr?.message }, '[crawler] route interception setup failed — downloads may occur');
+      }
     }
     // Wire rawPage to the real Playwright page from this CDP connection.
     // This gives us a true Playwright Page that supports all events (.on('response'), etc.)
@@ -1148,6 +1243,89 @@ export async function crawlProject(opts: {
       document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
     ).catch(() => -1);
 
+    // ── Pre-auth public-page discovery ───────────────────────────────────────
+    // The page is unauthenticated here. On sites where login redirects users
+    // away from the marketing/landing pages (e.g. "/" → app dashboard), BFS
+    // (which runs after auth) will never discover those public pages. We extract
+    // nav links from the current unauthenticated view and visit/capture each one
+    // before the session is applied, so the marketing pages are captured correctly.
+    const preAuthPublicCaptured = new Set<string>();
+    if (!opts.discoveryMode) {
+      const _appOrigin = new URL(opts.appUrl).origin;
+      // Simple URL normalizer for pre-auth use (canonicalize() declared later in scope)
+      const _norm = (u: string) => { try { const p = new URL(u); p.hash = ''; return p.href.replace(/\/$/, ''); } catch { return u; } };
+      const _startNorm = _norm(opts.appUrl);
+      const _authPathRe = /\/(sign.?in|sign.?up|log.?in|register|auth|oauth|sso|forgot.?password|reset.?password)(\/|$|\?)/i;
+      const _skipPathRe = /\/(terms|privacy|legal|cookie-policy|changelog|blog|imprint)([-/]|$)/i;
+
+      const publicNavLinks: string[] = await page.evaluate((originStr: string) => {
+        return Array.from(document.querySelectorAll('a[href]'))
+          .map((a: Element) => {
+            try {
+              const u = new URL((a as HTMLAnchorElement).getAttribute('href')!, window.location.href);
+              return u.origin === originStr ? u.href.split('#')[0] : '';
+            } catch { return ''; }
+          })
+          .filter((h: string, i: number, arr: string[]) => h && arr.indexOf(h) === i)
+          .slice(0, 30);
+      }, _appOrigin).catch(() => [] as string[]);
+
+      for (const link of publicNavLinks) {
+        if (_norm(link) === _startNorm) continue;
+        try {
+          const { pathname } = new URL(link);
+          if (_authPathRe.test(pathname) || _skipPathRe.test(pathname) || API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
+          await page.goto(link, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => {});
+          await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
+          const finalUrl = page.url();
+          const finalPath = new URL(finalUrl).pathname;
+          if (_authPathRe.test(finalPath)) continue; // redirected to auth — skip
+          if (_norm(finalUrl) === _startNorm) continue; // SPA catch-all redirect
+          // Read interactive elements first — SPA pages show 0 elements without auth.
+          // Only lock as pre-auth if the page actually has content; otherwise BFS handles it authenticated.
+          const preAuthElements = await readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>);
+          if (preAuthElements.length === 0) {
+            logger.info({ url: finalUrl }, '[crawler] pre-auth page has 0 elements — deferring to authenticated BFS');
+            continue;
+          }
+          // Detect auth walls that have a tiny number of elements (login button, back link, spinner).
+          // These pages redirect or gate on auth — main BFS must visit them with a live session.
+          // Label prefix or exact match — "Back to Learn" matches "back to", "Sign In" matches "sign in"
+          const AUTH_WALL_LABEL_RE = /^(sign.?in|log.?in|login|platform access|back\s+to|checking\s+auth|get started|create account|register|continue with|join now)/i;
+          // Also treat pages with only `a`-role elements (no inputs/buttons) as likely auth walls
+          const allAnchorOnly = preAuthElements.every(el => el.role === 'a');
+          const looksLikeAuthWall = preAuthElements.length <= 3 && (allAnchorOnly || preAuthElements.every(el => AUTH_WALL_LABEL_RE.test(el.label)));
+          if (looksLikeAuthWall) {
+            logger.info({ url: finalUrl, elements: preAuthElements.length, labels: preAuthElements.map(e => e.label) }, '[crawler] pre-auth auth wall — deferring to authenticated BFS');
+            continue;
+          }
+          const pathParts = finalPath.split('/').filter(Boolean);
+          const screenName = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, finalUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
+          if (!screen) continue;
+          const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
+          await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {});
+          const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
+          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl).catch(() => {});
+          // Save elements found in the pre-auth view
+          await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
+          for (const el of preAuthElements) {
+            await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, undefined, undefined, true, undefined, undefined, undefined).catch(() => {});
+          }
+          preAuthPublicCaptured.add(finalUrl);
+          logger.info({ url: finalUrl, screenId: screen.id, elements: preAuthElements.length }, '[crawler] pre-auth public page captured');
+        } catch (err: any) {
+          logger.warn({ url: link, err: err?.message }, '[crawler] pre-auth public page visit failed — skipping');
+        }
+      }
+
+      // Navigate back to start so auth phase and fingerprint comparison are correct
+      if (preAuthPublicCaptured.size > 0) {
+        await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 20_000 }).catch(() => {});
+        await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+      }
+    }
+
     // Dedicated logged-out/auth discovery phase. This runs before credentials or
     // storageState are applied, so public pages and auth recovery flows remain
     // visible even when the main crawl later becomes authenticated.
@@ -1159,6 +1337,18 @@ export async function crawlProject(opts: {
       }
     }
 
+    // Auth discovery opens OAuth sign-in popups (Google, GitHub, etc.) which fire CDP events that
+    // can permanently corrupt the Playwright isolated world. Verify the world is intact; if not,
+    // re-navigate to appUrl to force a fresh execution context before BFS begins.
+    {
+      const worldOk = await page.evaluate(() => true).catch(() => false);
+      if (!worldOk) {
+        logger.warn('[crawler] isolated world corrupted after auth discovery — re-navigating to recover');
+        await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => {});
+        await page.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => {});
+      }
+    }
+
     let didLogin = false; // track whether AI login was actually performed
 
     if (opts.authToken) {
@@ -1179,14 +1369,50 @@ export async function crawlProject(opts: {
         };
 
         if (state.cookies?.length) {
-          // page.context() is not available on Stagehand V3 proxy — use the secondary Playwright client
+          // Apply cookies to both the secondary Playwright context AND Stagehand's own context.
+          // Stagehand V3 uses a separate browser context from the CDP secondary client, so
+          // cookies added only to playwrightBrowser.contexts()[0] don't reach Stagehand's pages.
           const browserCtx = playwrightBrowser?.contexts()[0];
           if (browserCtx) {
             await browserCtx.addCookies(state.cookies as any);
           }
+          // Also apply to Stagehand's context via its page proxy
+          try {
+            const stagehandCtx = (page as any).context?.();
+            if (stagehandCtx && typeof stagehandCtx.addCookies === 'function') {
+              await stagehandCtx.addCookies(state.cookies as any);
+            }
+          } catch { /* Stagehand V3 proxy may not expose context() */ }
         }
 
         if (state.origins?.length) {
+          // Apply localStorage via addInitScript to ALL contexts:
+          // 1. Secondary Playwright context (rawPage navigations)
+          // 2. Stagehand's page directly (Stagehand V3 uses its own context separate from playwrightBrowser)
+          // Without #2, Stagehand's BFS navigations never get localStorage → pages load unauthenticated.
+          const originCtx = playwrightBrowser?.contexts()[0];
+          if (originCtx) {
+            await originCtx.addInitScript((origins: typeof state.origins) => {
+              for (const entry of (origins ?? [])) {
+                if (window.location.origin === entry.origin) {
+                  for (const item of entry.localStorage) {
+                    try { localStorage.setItem(item.name, item.value); } catch { /* quota/sandboxed */ }
+                  }
+                }
+              }
+            }, state.origins);
+          }
+          // Stagehand's page — covers all BFS navigations via Stagehand
+          await page.addInitScript((origins: typeof state.origins) => {
+            for (const entry of (origins ?? [])) {
+              if (window.location.origin === entry.origin) {
+                for (const item of entry.localStorage) {
+                  try { localStorage.setItem(item.name, item.value); } catch { /* quota/sandboxed */ }
+                }
+              }
+            }
+          }, state.origins);
+          // Also cover rawPage for the initial navigation below
           await rawPage.addInitScript((origins: typeof state.origins) => {
             for (const entry of (origins ?? [])) {
               if (window.location.origin === entry.origin) {
@@ -1396,8 +1622,17 @@ export async function crawlProject(opts: {
         [
           'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'ref', 'source', '_gl', 'fbclid', 'gclid',
           'ts', 'timestamp', 'cache', 'cacheBust', 'cache_bust', '_', 'nonce', 'session', 'sid',
+          // App content/state params — these change the content rendered within the same
+          // page template (a playground example, a search query, a tab selection) but do NOT
+          // represent structurally distinct pages. Not stripping them causes URL explosion:
+          // every unique ?cmd=DSN-XXXXX&example=... or ?q=foo is queued as a new screen.
+          'cmd', 'example', 'query', 'q', 'search', 'keyword', 'keywords',
+          'tab', 'view', 'mode', 'sort', 'order', 'direction', 'orderby',
+          'page', 'p', 'offset', 'limit', 'per_page', 'size', 'rows',
+          // Export format params — same endpoint, different output format, not a new screen
+          'format', 'output', 'export_format', 'download_format', 'fileformat', 'file_format',
           // WooCommerce / e-commerce action params
-          'add-to-cart', 'add_to_wishlist', 'quantity', 'wc-ajax', 'orderby',
+          'add-to-cart', 'add_to_wishlist', 'quantity', 'wc-ajax',
           'min_price', 'max_price', 'removed_item', 'restore', 'remove_item',
           'undo_item', 'coupon_code', 'apply_coupon', 'added-to-cart',
           'wc_error', 'wc_notice', 'wc_notice_type', 'update_cart',
@@ -1440,10 +1675,14 @@ export async function crawlProject(opts: {
     const gitInfo = getGitInfo();
 
     const visited = new Map<string, string>(); // key = canonical URL
+    const crawlWarnings: string[] = []; // non-fatal step failures surfaced in job result
     // Pre-populate from a previous failed job so already-captured URLs are skipped
     if (opts.skipUrls?.length) {
       for (const u of opts.skipUrls) visited.set(canonicalize(u), 'skipped');
     }
+    // Pre-auth public pages already captured above — mark visited so BFS (post-auth)
+    // doesn't overwrite them with authenticated redirects showing app content instead.
+    for (const u of preAuthPublicCaptured) visited.set(canonicalize(u), 'pre-auth');
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
     let lastEmittedPct = 20; // floor above auth-discovery + login phases (max ~19%)
@@ -1504,6 +1743,10 @@ export async function crawlProject(opts: {
             .filter((s: any) => s.classification !== 'REACHABLE')
             .map((s: any) => s.id as string),
         );
+        // Snapshot queued set BEFORE filter so canonical dedup works correctly during filter.
+        // Without this, multiple prior screens with the same canonical (e.g. playground?cmd=X1,
+        // playground?cmd=X2 → same canonical) all pass the filter since queued isn't updated yet.
+        const seenReseedCanon = new Set<string>(queued);
         const priorScreens = (priorMap?.screens ?? [])
           .filter((s: any) => s.status !== 'PENDING_REMOVAL')
           .filter((s: any) => {
@@ -1515,10 +1758,15 @@ export async function crawlProject(opts: {
               if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
               // Don't re-seed generated dynamic routes — they produce duplicate templates.
               if (isGeneratedDynamicPath(pathname)) return false;
+              // Don't re-seed API/export/download paths — they were misidentified as screens.
+              if (!isUsefulNavigationUrl(u, startUrl)) return false;
+              // Canonical dedup: mark seen immediately during filter so duplicates are caught.
+              const canon = canonicalize(u);
+              if (seenReseedCanon.has(canon)) return false;
+              seenReseedCanon.add(canon);
               return true;
             } catch { return false; }
-          })
-          .filter((s: any) => !queued.has(canonicalize(s.url as string)));
+          });
         let orphanSeedCount = 0;
         for (const s of priorScreens) {
           const u = s.url as string;
@@ -1580,6 +1828,7 @@ export async function crawlProject(opts: {
             try {
               const u = new URL(sUrl);
               if (u.origin !== sitemapOrigin) continue;
+              if (API_PATH_RE.test(u.pathname) || EXPORT_PATH_RE.test(u.pathname) || DOWNLOAD_EXT_RE.test(u.pathname)) continue;
               await page.goto(sUrl, { waitUntil: 'domcontentloaded', timeout: 12_000 });
               await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
               await waitForStableDOM(page);
@@ -1664,6 +1913,9 @@ export async function crawlProject(opts: {
     // that slip through canonicalize() if any new filter param type is encountered.
     let totalAttemptsThisSegment = 0;
     const maxAttempts = Math.max(segmentSize * 4, 200);
+    // Count consecutive login-wall redirects — if too many, abort early (auth-wall site, no creds)
+    let loginWallRedirectCount = 0;
+    const LOGIN_WALL_ABORT_THRESHOLD = Math.max(8, Math.ceil(segmentSize * 0.4));
 
     let pendingSessionRestore: (() => Promise<void>) | null = null;
 
@@ -1687,6 +1939,8 @@ export async function crawlProject(opts: {
         try {
           const { pathname } = new URL(url);
           if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+          // Skip API/export paths — these return data files, not navigable pages
+          if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
         } catch { continue; }
 
         // Claim URL before first await — prevents duplicate processing across parallel workers
@@ -1701,6 +1955,28 @@ export async function crawlProject(opts: {
           // Wait for SPA JS to render components before extracting elements or running soft-404 check.
           // domcontentloaded fires before JS executes — networkidle ensures React/Vue/Angular hydration.
           await workerPage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+          // Wait for DOM to stop mutating (React/Vue render cycles) before element extraction.
+          await waitForStableDOM(workerPage, 6_000, 400).catch(() => {});
+          // Dismiss cookie banners / GDPR overlays so underlying elements are accessible.
+          await dismissOverlays(workerPage).catch(() => {});
+
+          // Auth wall detection: if navigation redirected to a login page, skip this URL.
+          // Lite worker cannot re-authenticate (no Stagehand access) — mark and skip.
+          const lwCurrentUrl = workerPage.url();
+          const lwPath = (() => { try { return new URL(lwCurrentUrl).pathname.toLowerCase(); } catch { return ''; } })();
+          const lwIsAuthWall = /^\/(login|signin|sign-in|auth|authenticate|session\/new|account\/login|user\/login)/.test(lwPath) && lwCurrentUrl !== url;
+          const lwHasLoginForm = lwIsAuthWall ? false : await workerPage.evaluate(() => {
+            const inputs = document.querySelectorAll('input[type="password"]');
+            return inputs.length > 0;
+          }).catch(() => false);
+          if (lwIsAuthWall) {
+            logger.warn({ url, landed: lwCurrentUrl }, '[lite-worker] auth wall detected — queuing for main worker re-auth');
+            visited.set(canonUrl, '_auth_wall_');
+            if (!revisitQueue.includes(url)) revisitQueue.push(url);
+            continue;
+          }
+          // lwHasLoginForm (direct sign-in/register page): process normally — capture the form elements.
+          // Do NOT add to revisitQueue; P3a cannot do anything useful with a login page revisit.
 
           // Soft-404 detection for lite-worker: skip SPA catch-all URLs that mirror homepage
           if (homeFingerprintElemCount >= 0 && url !== startUrl) {
@@ -1725,8 +2001,23 @@ export async function crawlProject(opts: {
 
           const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
 
+          let liteElements = await domFallbackElements(workerPage).catch(() => [] as Awaited<ReturnType<typeof domFallbackElements>>);
+          if (liteElements.length === 0) {
+            // domFallbackElements returned nothing — try broader visible-element selector as last resort.
+            liteElements = await workerPage.evaluate(() => {
+              const sel = 'button,a[href],input,select,textarea,[role],[tabindex="0"],[onclick],[data-action]';
+              return Array.from(document.querySelectorAll(sel))
+                .filter((el: any) => el.offsetParent !== null)
+                .slice(0, 150)
+                .map((el: any) => ({
+                  meaning: el.getAttribute('aria-label') || el.textContent?.trim().slice(0, 100) || el.tagName.toLowerCase(),
+                  role: el.getAttribute('role') || el.tagName.toLowerCase(),
+                  expectedData: null, notes: null, boundingRect: null, ariaState: null, parentLandmark: null,
+                }));
+            }).catch(() => []) as any[];
+          }
           const [elements] = await Promise.all([
-            domFallbackElements(workerPage).catch(() => [] as Awaited<ReturnType<typeof domFallbackElements>>),
+            Promise.resolve(liteElements),
             workerPage.screenshot({ path: shotPath, fullPage: true })
               .then(async () => {
                 const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
@@ -1735,25 +2026,184 @@ export async function crawlProject(opts: {
               .catch(() => {}),
           ]);
 
+          // Nav-link pass — merged into main save cycle so clearScreenElements covers them too.
+          // extractElements (main BFS) never runs for lite-worker screens, so nav/footer roles
+          // must be captured here. Running before clear+save prevents additive accumulation.
+          const _navEls = await (workerPage.evaluate as any)(() => {
+            if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
+            const seen = new Set<string>();
+            const items: Array<{meaning: string; role: string; notes: string}> = [];
+            function isV(el: Element) {
+              const s = window.getComputedStyle(el);
+              return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
+            }
+            // Wide nav selector: covers semantic nav, ARIA, header links, sidebar (aside/[role=complementary]),
+            // and common class-based sidebars used by admin/dashboard pages.
+            const NAV_SEL = [
+              'nav a', '[role="navigation"] a', 'header a', 'footer nav a',
+              '[aria-label*="navigation" i] a', '[aria-label*="sidebar" i] a',
+              'aside a', '[role="complementary"] a',
+              '[class*="sidebar" i] a', '[class*="side-nav" i] a', '[class*="sidenav" i] a',
+              '[class*="nav-menu" i] a', '[class*="navmenu" i] a', '[data-nav] a',
+            ].join(', ');
+            document.querySelectorAll(NAV_SEL).forEach((a: any) => {
+              if (!isV(a)) return;
+              const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+              const href = a.href ?? '';
+              if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
+              let landmark = 'Site Navigation';
+              let cur: any = a.parentElement;
+              while (cur) {
+                const t = cur.tagName?.toLowerCase();
+                const r = cur.getAttribute?.('role')?.toLowerCase();
+                const cl = (cur.className ?? '').toLowerCase();
+                if (t === 'header' || r === 'banner') { landmark = 'Header'; break; }
+                if (t === 'footer' || r === 'contentinfo') { landmark = 'Footer'; break; }
+                if (t === 'aside' || r === 'complementary' || cl.includes('sidebar') || cl.includes('sidenav') || cl.includes('side-nav')) { landmark = 'Sidebar'; break; }
+                if (t === 'nav' || r === 'navigation') { landmark = cur.getAttribute('aria-label')?.trim() || 'Site Navigation'; break; }
+                cur = cur.parentElement;
+              }
+              const key = `nav|${label.toLowerCase()}|${href}`;
+              if (!seen.has(key)) { seen.add(key); items.push({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark }) }); }
+            });
+            let fc = 0;
+            document.querySelectorAll('footer a, [role="contentinfo"] a, [class*="footer" i] a').forEach((a: any) => {
+              if (fc >= 20 || !isV(a)) return;
+              const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+              const href = a.href ?? '';
+              if (!label || !href) return;
+              const key = `footer|${label.toLowerCase()}|${href}`;
+              if (!seen.has(key)) { seen.add(key); items.push({ meaning: label, role: 'footer-link', notes: JSON.stringify({ href: href.slice(0, 150) }) }); fc++; }
+            });
+            // Grandparent-grouping fallback: detects sidebar navigation where each link is
+            // individually wrapped (e.g. <div class="flex"><a>link</a></div> × many siblings).
+            // Runs ALWAYS (not gated on 0 results) so header links don't block sidebar detection.
+            // Deduped against hrefs already captured by the semantic pass above.
+            {
+              const navHrefsSeen = new Set<string>();
+              for (const it of items) {
+                try { const h = JSON.parse(it.notes ?? '{}').href; if (h) navHrefsSeen.add(h); } catch { /* skip */ }
+              }
+              const gpGroups = new Map<Element, Array<{label: string; href: string}>>();
+              document.querySelectorAll('a[href]').forEach((a: any) => {
+                if (!isV(a)) return;
+                const href = a.href ?? '';
+                if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsSeen.has(href)) return;
+                const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+                if (!label || label.length > 40) return;
+                const parent = a.parentElement;
+                const gp = parent?.parentElement;
+                if (!parent || !gp) return;
+                // Only match when parent wraps exactly 1 anchor — the sidebar <div><a> pattern
+                if (parent.querySelectorAll('a').length !== 1) return;
+                const gpTag = gp.tagName.toLowerCase();
+                if (['body', 'html', 'form'].includes(gpTag)) return;
+                const arr = gpGroups.get(gp) ?? [];
+                arr.push({ label, href });
+                gpGroups.set(gp, arr);
+              });
+              for (const [, links] of gpGroups.entries()) {
+                if (links.length >= 4) {
+                  for (const { label, href } of links) {
+                    const k = `cluster|${label.toLowerCase()}|${href}`;
+                    if (!seen.has(k)) { seen.add(k); items.push({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark: 'Sidebar' }) }); }
+                  }
+                }
+              }
+            }
+            return items;
+          }).catch(() => [] as any[]);
+
+          const allElements = [
+            ...elements,
+            ..._navEls.map((ne: any) => ({ ...ne, expectedData: undefined, boundingRect: undefined, ariaState: undefined, parentLandmark: undefined })),
+          ];
+
+          // Persist lite-worker elements to DB — without this, 80% of screens have 0 DB elements.
+          if (allElements.length > 0) {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
+            for (let _ei = 0; _ei < allElements.length; _ei += 4) {
+              const _batch = allElements.slice(_ei, _ei + 4);
+              await Promise.all(_batch.map((el: any) =>
+                AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {})
+              ));
+            }
+            logger.info({ url, screenId: screen.id, elements: elements.length, navLinks: _navEls.length }, '[lite-worker] screen captured');
+          } else if (!revisitQueue.includes(url)) {
+            // 0-element: queue for main BFS re-visit with full Stagehand AI extraction
+            logger.warn({ url, screenId: screen.id }, '[lite-worker] 0 elements — queued for P3a revisit');
+            revisitQueue.push(url);
+          }
+
+          // Non-mutating probes for lite-worker screens: shadow DOM, a11y tree, service worker.
+          // Awaited with 15s total timeout — these are read-only and safe on the shared worker page.
+          if (screen?.id) {
+            const [shadowRes, a11yRes, swRes] = await Promise.race([
+              Promise.allSettled([
+                crawlShadowDom(workerPage),
+                crawlAccessibilityTree(workerPage),
+                inspectServiceWorker(workerPage),
+              ]),
+              new Promise<PromiseSettledResult<never>[]>((resolve) =>
+                setTimeout(() => resolve([
+                  { status: 'rejected' as const, reason: 'timeout' },
+                  { status: 'rejected' as const, reason: 'timeout' },
+                  { status: 'rejected' as const, reason: 'timeout' },
+                ]), 15_000)
+              ),
+            ]);
+            const litePayload: Record<string, unknown> = { screenId: screen.id, url };
+            if (shadowRes.status === 'fulfilled' && shadowRes.value && (shadowRes.value as any).shadowHostCount > 0)
+              litePayload.shadowDom = shadowRes.value;
+            if (a11yRes.status === 'fulfilled' && a11yRes.value)
+              litePayload.accessibilityTree = a11yRes.value;
+            if (swRes.status === 'fulfilled' && (swRes.value as any).supported)
+              litePayload.serviceWorker = swRes.value;
+            if (Object.keys(litePayload).length > 2) {
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId, type: 'PER_SCREEN_PROBES', payload: litePayload,
+              }).catch(() => {});
+            }
+          }
+
           const nextLinks: string[] = await (workerPage.evaluate as any)((originStr: string) => {
-            return Array.from(document.querySelectorAll('a[href]'))
+            return Array.from(document.querySelectorAll('a[href],[data-href]'))
               .map((a: any) => {
-                try { return new URL(a.getAttribute('href')!, window.location.href).href; }
+                const raw = a.getAttribute('href') || a.getAttribute('data-href') || '';
+                try { return new URL(raw, window.location.href).href; }
                 catch { return ''; }
               })
-              .filter((h: string) => h && new URL(h).origin === originStr && !h.includes('#'))
+              .filter((h: string) => {
+                if (!h) return false;
+                try {
+                  const u = new URL(h);
+                  if (u.origin !== originStr) return false;
+                  // Keep hash-router paths (#/route, #!/route) but drop plain fragment anchors (#section)
+                  if (u.hash && !u.hash.startsWith('#/') && !u.hash.startsWith('#!/')) return false;
+                  return true;
+                } catch { return false; }
+              })
               .slice(0, 200);
           }, origin).catch(() => []);
 
           for (const link of nextLinks) {
+            if (!isUsefulNavigationUrl(link, origin)) continue;
             const cLink = canonicalize(link);
-            if (!visited.has(cLink)) {
+            if (!visited.has(cLink) && !queued.has(cLink)) {
+              queued.add(cLink);
               queue.push(link);
               edges.push({ fromUrl: url, toUrl: link });
             }
           }
 
           discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+          // Lite workers also advance the progress bar — without this 4/5 workers are invisible.
+          const lwEffMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+          maxSeenTotal = Math.max(maxSeenTotal, lwEffMax);
+          const lwRawPct = 20 + Math.round(Math.min(57, (visited.size / Math.max(1, maxSeenTotal)) * 57));
+          const lwPct = Math.max(lastEmittedPct, lwRawPct);
+          lastEmittedPct = lwPct;
+          await onProgress?.(lwPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`).catch(() => {});
         } catch (err: any) {
           logger.warn({ url, err: err?.message ?? String(err) }, '[lite-worker] URL processing failed');
           discovered.push({ url, error: err?.message ?? String(err) });
@@ -1762,7 +2212,7 @@ export async function crawlProject(opts: {
     }
 
     const liteWorkerPromises: Promise<void>[] = [];
-    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser) {
+    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser && !opts.discoveryMode) {
       const pwCtx = playwrightBrowser.contexts()[0];
       for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
         const wp = await pwCtx.newPage();
@@ -1793,11 +2243,14 @@ export async function crawlProject(opts: {
 
       const url = queue.shift()!;
       const canonUrl = canonicalize(url);
-      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size + 1}/${segmentSize}` }, '[crawler] visiting URL');
       const apiObservationStartIndex = capturedApiEndpoints.length;
 
       // Skip if already visited (canonical match — catches ID-parameterized duplicates)
       if (visited.has(canonUrl)) continue;
+      // Claim immediately before any await — prevents lite workers from picking up the same URL
+      // concurrently while this iteration is suspended (race that causes repeated visits).
+      visited.set(canonUrl, '_pending_');
+      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size}/${segmentSize}` }, '[crawler] visiting URL');
 
       // robots.txt compliance — skip disallowed URLs
       if (robotsCache && !await robotsCache.isAllowed(url)) {
@@ -1819,6 +2272,13 @@ export async function crawlProject(opts: {
           console.log(`[crawler] skip generated dynamic route: ${url}`);
           continue;
         }
+        // Skip API/export/download paths — navigating to these triggers file downloads.
+        // The main BFS uses Stagehand's context which has no route interception.
+        if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) {
+          console.log(`[crawler] skip API/export/download path: ${url}`);
+          visited.set(canonUrl, '_skipped_api_');
+          continue;
+        }
       } catch { /* invalid url, let it fall through */ }
 
       try {
@@ -1914,13 +2374,18 @@ export async function crawlProject(opts: {
           }
           rawPage.off('response', onRedirectResponse);
           if (redirectHops.length > 0) {
+            const isCircularRedirect = new Set(redirectHops.map(h => h.from)).size < redirectHops.length;
             redirectRegistry.register({
               originalUrl: url, finalUrl: page.url(),
               hops: redirectHops.map((h, i) => ({ ...h, durationMs: i === 0 ? Date.now() - navStart : 0 })),
-              isCircular: new Set(redirectHops.map(h => h.from)).size < redirectHops.length,
+              isCircular: isCircularRedirect,
               hopCount: redirectHops.length,
               totalDurationMs: Date.now() - navStart,
             });
+            if (isCircularRedirect) {
+              logger.warn({ url, hops: redirectHops.map(h => h.from) }, '[crawler] circular redirect detected — skipping URL');
+              continue;
+            }
           }
           // Update exponential moving average for AutoThrottle
           const navMs = Date.now() - navStart;
@@ -1939,8 +2404,54 @@ export async function crawlProject(opts: {
         // 3. Modal capture — screenshot app dialogs BEFORE dismissal (they are testable UI states)
         // 4. Overlay dismissal — removes cookie banners/modals before element extraction
         // 5. Lazy-load scroll — triggers IntersectionObserver for off-screen images/components
-        await page.waitForLoadState('load', { timeout: 8_000 }).catch(() => { });
-        await waitForStableDOM(page);
+        await (rawPage ?? page).waitForLoadState('load', { timeout: 8_000 }).catch(() => { });
+        await waitForStableDOM(page, 8_000, 400, rawPage ?? undefined);
+        // If DOM is sparse after stability check (< 15 interactive elements), SPA may still be
+        // fetching data — skeleton is DOM-stable but empty. Extra networkidle pass catches it.
+        const sparseCheck = await (rawPage ?? page).evaluate(() =>
+          document.querySelectorAll('button,a,input,select,textarea,[role="button"],[aria-label]').length
+        ).catch(() => 99);
+        if (sparseCheck < 15) {
+          await (rawPage ?? page).waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => {});
+          await waitForStableDOM(page, 4_000, 300, rawPage ?? undefined);
+        }
+        // Wait for SPA auth-check / loading spinners to resolve before screenshotting.
+        // SPAs often show a "Checking authentication..." or "Loading..." state after networkidle
+        // while client-side auth verifies tokens. waitForStableDOM catches DOM mutations but
+        // returns as soon as the spinner DOM is stable — this catches the spinner itself.
+        // Stagehand V3 proxy may not expose waitForFunction — use try/catch, not .catch(),
+        // because calling an undefined method throws synchronously before a Promise is created.
+        try {
+          const spinnerPage = rawPage ?? page;
+          if (typeof spinnerPage.waitForFunction === 'function') {
+            await spinnerPage.waitForFunction(() => {
+              const body = document.body;
+              if (!body) return true;
+              const text = body.innerText?.toLowerCase() ?? '';
+              // Text-based loading indicators
+              if (
+                text.includes('checking authentication') ||
+                text.includes('checking auth') ||
+                text.includes('authenticating') ||
+                text.includes('verifying session') ||
+                text.includes('please wait') ||
+                text.includes('loading data') ||
+                (text.includes('loading') && text.length < 200)
+              ) return false;
+              // ARIA-based: aria-busy on any visible element
+              const busyEl = body.querySelector('[aria-busy="true"]');
+              if (busyEl && (busyEl as HTMLElement).offsetParent !== null) return false;
+              // Data attributes: React Query / SWR / TanStack in-flight indicators
+              if (body.querySelector('[data-loading="true"],[data-pending="true"],[data-fetching="true"]')) return false;
+              // Indeterminate progress bars
+              if (body.querySelector('progress:not([value])')) return false;
+              // Common CSS class patterns for loading skeletons
+              const skeleton = body.querySelector('.skeleton,.skeleton-loader,.shimmer,.pulse-loader,.content-loader');
+              if (skeleton && (skeleton as HTMLElement).offsetParent !== null) return false;
+              return true;
+            }, { timeout: 12_000 });
+          }
+        } catch { /* timed out or unsupported — screenshot whatever is there */ }
 
         if (!opts.discoveryMode) {
           // Capture app-generated modals BEFORE dismissing — cookie/GDPR banners excluded.
@@ -1958,20 +2469,25 @@ export async function crawlProject(opts: {
               const modalHash = url.replace(/[^a-z0-9]/gi, '').slice(-12);
               const modalShotPath = path.join(opts.screenshotDir ?? '/tmp', `modal-${modalHash}.png`);
               await page.screenshot({ path: modalShotPath, fullPage: false }).catch(() => { });
-              const modalName = await page.evaluate(() => {
+              const modalName = await (rawPage ?? page).evaluate(() => {
                 const h = document.querySelector('[role="dialog"] h1,[role="dialog"] h2,[role="dialog"] [role="heading"]');
                 return h?.textContent?.trim() ?? 'Modal';
               }).catch(() => 'Modal');
               const modalScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, modalName, url, opts.jobId, discoveryMethodFor(url)).catch(() => null);
               if (modalScreen) {
                 await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, 'DESKTOP').catch(() => { });
-                logger.info({ url, name: modalName }, '[crawler] captured app modal before dismiss');
+                // Extract elements from the modal BEFORE it is dismissed — use CDP path via rawPage
+                const modalElements = await domFallbackElements(page, rawPage ?? undefined).catch(() => []);
+                for (const el of modalElements) {
+                  AppBrain.saveElement(opts.tenantId, opts.projectId, modalScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {});
+                }
+                logger.info({ url, name: modalName, elementCount: modalElements.length }, '[crawler] captured app modal before dismiss');
               }
             }
           } catch { /* non-fatal */ }
           await dismissOverlays(page);
           // Enhanced infinite scroll — handles lazy-load, infinite lists, and virtual scrollers
-          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }).catch(() => { });
+          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }, rawPage ?? undefined).catch(() => { });
         }
         if (cdpSession?.off) cdpSession.off('Network.responseReceived', secHeadersCdp);
         wsDetector.detach();
@@ -2002,6 +2518,22 @@ export async function crawlProject(opts: {
           BFS_ERROR_PATH.test(new URL(landedUrl).pathname) || BFS_LOGIN_PATH.test(new URL(landedUrl).pathname)
         );
         if (redirectedToError) {
+          // Track login-wall redirects: if the site sends every URL to /login without credentials,
+          // abort early rather than burning all maxAttempts on the same login page.
+          if (BFS_LOGIN_PATH.test(new URL(landedUrl).pathname)) {
+            loginWallRedirectCount++;
+            if (!opts.credentials && loginWallRedirectCount >= LOGIN_WALL_ABORT_THRESHOLD) {
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                type: 'AUTH_FAILURE',
+                payload: {
+                  url, landed: landedUrl,
+                  reason: `Login wall detected: ${loginWallRedirectCount} consecutive URLs redirected to login. Add credentials in project settings to crawl authenticated pages.`,
+                },
+              }).catch(() => {});
+              throw new Error(`[AUTH_FAILURE] Login wall detected: ${loginWallRedirectCount} URLs redirected to the login page. Add credentials in project settings and retry the crawl.`);
+            }
+          }
           // We asked for /dashboard but landed on /session-expired — session expired mid-crawl.
           logger.warn({ requested: url, landed: landedUrl }, '[crawler] Mid-crawl redirect to auth/error — attempting re-auth');
           let reauthed = false;
@@ -2016,8 +2548,8 @@ export async function crawlProject(opts: {
             if (reauthed) {
               logger.info({ url }, '[crawler] Re-auth succeeded — retrying URL');
               await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(() => { });
-              await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
-              await waitForStableDOM(page);
+              await (rawPage ?? page).waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
+              await waitForStableDOM(page, 8_000, 400, rawPage ?? undefined);
               const afterReauthUrl = page.url();
               if (BFS_ERROR_PATH.test(new URL(afterReauthUrl).pathname) || BFS_LOGIN_PATH.test(new URL(afterReauthUrl).pathname)) {
                 logger.warn({ url }, '[crawler] Still redirecting after re-auth — skipping this URL');
@@ -2033,6 +2565,13 @@ export async function crawlProject(opts: {
               throw new Error('[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.');
             }
           } else {
+            // Discovery mode: skip auth-required URLs and continue discovering other links.
+            // Throwing here aborts the entire BFS — in discovery mode we want to continue
+            // visiting other queued URLs (e.g. sitemap seeds) that may not require auth.
+            if (opts.discoveryMode) {
+              logger.info({ url, landed: landedUrl }, '[discover] skipping auth-required URL — no credentials, continuing BFS');
+              continue;
+            }
             logger.warn('[crawler] Session expired mid-crawl, no credentials to recover — skipping URL');
             AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
               jobId: opts.jobId,
@@ -2047,6 +2586,8 @@ export async function crawlProject(opts: {
         if (!redirectedToError && landedUrl !== url) {
           logger.info({ requested: url, landed: landedUrl }, '[crawler] navigated with redirect');
         }
+        // Successful non-login page visit: reset the consecutive login-wall counter
+        if (!redirectedToError) loginWallRedirectCount = 0;
 
         // Discovery mode fast-path: record URL + extract DOM links only; skip screenshots/DB/AI.
         // IMPORTANT: DOM hash dedup is intentionally DISABLED in discovery mode — SPAs use identical
@@ -2194,8 +2735,10 @@ export async function crawlProject(opts: {
         }
 
         // Normal crawl mode: DOM hash dedup — detect duplicate-structure pages (SPA 404s, template clones).
+        // Skip dedup when hash is empty (computeDomHash failed) — two failed-hash pages would
+        // incorrectly match each other and the second would be silently dropped.
         const preDomHash = await computeDomHash(page);
-        if (seenDomHashes.has(preDomHash)) {
+        if (preDomHash !== '' && seenDomHashes.has(preDomHash)) {
           console.log(`[crawler] skip ${url} — duplicate DOM structure (same content as prior page)`);
           continue;
         }
@@ -2322,7 +2865,7 @@ export async function crawlProject(opts: {
             }
 
             // Append significant query params
-            const SIG_PARAMS = ['product_cat', 'category', 'cat', 'type', 'tag', 'brand', 'filter', 'tab', 'section', 'view'];
+            const SIG_PARAMS = ['product_cat', 'category', 'cat', 'type', 'tag', 'brand', 'filter', 'tab', 'section', 'view', 'play', 'mode', 'id', 'page'];
             const paramSuffix = SIG_PARAMS
               .map((p) => { const v = u.searchParams.get(p); return v ? ` · ${v.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())}` : ''; })
               .find(Boolean) ?? '';
@@ -2345,11 +2888,12 @@ export async function crawlProject(opts: {
                 const candidate = parts[0].trim();
                 if (candidate.length > 2 && !/^\d[\d\s]*$/.test(candidate)) return candidate;
               }
-              // Single-segment title — only use if multi-word and not brand-name-like
+              // Single-segment title — only use if multi-word, not brand-name-like, and not the generic SPA app title
               const candidate = parts[0].trim();
               const isBrand = candidate.split(/\s+/).length === 1 &&
                 (candidate.toLowerCase().includes(brandName) || candidate.toLowerCase() === brandName);
-              if (!isBrand && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
+              const isSiteGenericTitle = homeFingerprintTitle !== '' && candidate === homeFingerprintTitle;
+              if (!isBrand && !isSiteGenericTitle && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
                 return candidate;
               }
             }
@@ -2467,31 +3011,33 @@ export async function crawlProject(opts: {
           logger.info({ url, cachedCount: cachedLinks.length }, '[crawler] link-discovery: using cached recipe — AI will still run and merge');
         }
 
-        // Emit progress now — navigation is done and the page is stable.
-        // This fires before the 30-90s parallel screenshot+AI phase, preventing a silent gap in the UI.
-        // Phase 1 (link discovery): 5–20% over first 60s of wall-clock time so users see movement
-        // before visited.size grows enough to drive the 20–80% phase.
-        const elapsedDiscoverySec = (Date.now() - crawlLoopStart) / 1000;
-        const discoveryPct = Math.round(5 + Math.min(15, (elapsedDiscoverySec / 60) * 15));
-        const preCapturePct = Math.max(lastEmittedPct, discoveryPct);
-        lastEmittedPct = preCapturePct;
-        await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
-
-        // P3a: Async content detection — get raw DOM element count before AI extraction.
-        // The AI extraction takes 30-90s; any async-loaded content will be visible after it completes.
-        // Compare pre vs post count to detect dashboards that populate after initial load.
+        // Emit progress now — navigation done, page stable.
+        // Use the same visited-count formula as the post-capture block so progress
+        // advances on EVERY URL visit (not just on successful screen saves).
+        {
+          const effMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+          maxSeenTotal = Math.max(maxSeenTotal, effMax);
+          const rawPrePct = 20 + Math.round(Math.min(57, (visited.size / Math.max(1, maxSeenTotal)) * 57));
+          const preCapturePct = Math.max(lastEmittedPct, rawPrePct);
+          lastEmittedPct = preCapturePct;
+          await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
+        }
+
+        await waitForScreenshotReady(page, rawPage ?? undefined);
+
+        // P3a: Async content detection — count AFTER networkidle+lazy-load so lazy-fetched content
+        // is included. Previously captured before waitForScreenshotReady which caused AI skip logic
+        // to see skeleton element counts instead of real populated counts.
         const preDomCount = await page.evaluate(() =>
           document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
         ).catch(() => 0);
-
-        await waitForScreenshotReady(page);
         await onProgress?.(lastEmittedPct, `Capturing ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
 
         // FM-4: capture device pixel ratio for coordinate scaling in element highlighting
-        const pageDpr: number = await page.evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);
+        const pageDpr: number = await (rawPage ?? page).evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);
 
         // Extract elements first so we can highlight them in the screenshot
-        const elements = await extractElements(stagehand, page, elementLLM ?? undefined).catch((err: any) => {
+        const elements = await extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch((err: any) => {
           console.error('[crawler] extractElements failed:', err);
           return [] as Awaited<ReturnType<typeof extractElements>>;
         });
@@ -2550,8 +3096,8 @@ export async function crawlProject(opts: {
             .catch((err: any) => console.error('[crawler] page.screenshot failed:', err)),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
-          // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
-          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < aiMinElements)
+          // Note: preDomCount < aiMinElements guard removed — it was backwards: sparse pages (SPAs) need AI most.
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined)
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
@@ -2582,7 +3128,7 @@ export async function crawlProject(opts: {
               }
               return fallback;
             }),
-          readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>),
+          readScreen(rawPage ?? page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>),
         ]);
 
         // Cleanup highlights
@@ -2601,6 +3147,115 @@ export async function crawlProject(opts: {
           }
         }).catch(() => { });
 
+        // Per-screen probes: split read-only (awaited, safe on main page) vs mutating (isolated page).
+        // READ-ONLY probes are awaited before BFS navigates — DOM must reflect current screen.
+        // MUTATING probes (dropdowns/CRUD/filters/advanced) open a fresh page to isolate state changes.
+        {
+          // Resolve browser context from rawPage (actual Playwright page, not Stagehand proxy).
+          const safeContext: import('playwright').BrowserContext | null =
+            typeof (rawPage as any)?.context === 'function'
+              ? (rawPage as any).context()
+              : typeof (page as any)?.context === 'function'
+              ? (page as any).context()
+              : playwrightBrowser?.contexts()[0] ?? null;
+
+          // --- Read-only probes: awaited with 25s timeout, never mutate page state ---
+          const roTimeout = new Promise<PromiseSettledResult<never>[]>((resolve) =>
+            setTimeout(() => resolve([
+              { status: 'rejected' as const, reason: 'probe-timeout' },
+              { status: 'rejected' as const, reason: 'probe-timeout' },
+              { status: 'rejected' as const, reason: 'probe-timeout' },
+              { status: 'rejected' as const, reason: 'probe-timeout' },
+              { status: 'rejected' as const, reason: 'probe-timeout' },
+            ]), 25_000)
+          );
+          const [nestedCtxRes, shadowRes, a11yRes, perfRes, swRes] = await Promise.race([
+            Promise.allSettled([
+              crawlNestedContexts(rawPage ?? page, safeContext, url),
+              crawlShadowDom(rawPage ?? page),
+              crawlAccessibilityTree(rawPage ?? page),
+              collectPerfMetrics(rawPage ?? page, url),
+              inspectServiceWorker(rawPage ?? page),
+            ]),
+            roTimeout,
+          ]);
+
+          const roPayload: Record<string, unknown> = { screenId: screen.id, url };
+          if (nestedCtxRes.status === 'fulfilled' && nestedCtxRes.value && (nestedCtxRes.value as any).totalContextsFound > 0)
+            roPayload.nestedContexts = nestedCtxRes.value;
+          if (shadowRes.status === 'fulfilled' && shadowRes.value && (shadowRes.value as any).shadowHostCount > 0)
+            roPayload.shadowDom = shadowRes.value;
+          if (a11yRes.status === 'fulfilled' && a11yRes.value)
+            roPayload.accessibilityTree = a11yRes.value;
+          if (perfRes.status === 'fulfilled' && (perfRes.value as any).requestCount > 0)
+            roPayload.performance = perfRes.value;
+          if (swRes.status === 'fulfilled' && (swRes.value as any).supported)
+            roPayload.serviceWorker = swRes.value;
+          if (Object.keys(roPayload).length > 2) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId, type: 'PER_SCREEN_PROBES', payload: roPayload,
+            }).catch(() => {});
+          }
+
+          // --- Mutating probes (interactive): isolated page, fire-and-forget, only when interactive ---
+          if (screen?.id && opts.interactive !== false && safeContext) {
+            void (async () => {
+              let probePage: import('playwright').Page | null = null;
+              try {
+                // Cap concurrent probe pages: if too many already open, skip to prevent resource exhaustion.
+                const openProbePages = safeContext.pages().length;
+                if (openProbePages > 8) {
+                  logger.warn({ url, openProbePages }, '[crawler] probe page cap reached — skipping mutating probes for this screen');
+                  return;
+                }
+                probePage = await safeContext.newPage();
+                await probePage.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => {});
+                await probePage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+                await waitForStableDOM(probePage as any, 4_000, 400).catch(() => {});
+
+                const muTimeout = new Promise<PromiseSettledResult<never>[]>((resolve) =>
+                  setTimeout(() => resolve([
+                    { status: 'rejected' as const, reason: 'probe-timeout' },
+                    { status: 'rejected' as const, reason: 'probe-timeout' },
+                    { status: 'rejected' as const, reason: 'probe-timeout' },
+                    { status: 'rejected' as const, reason: 'probe-timeout' },
+                  ]), 60_000)
+                );
+                const [dropEnumRes, crudRes, filterRes, advancedRes] = await Promise.race([
+                  Promise.allSettled([
+                    enumerateDropdowns(probePage),
+                    probeCrudFlows(probePage, name),
+                    crawlFiltersAndPagination(probePage),
+                    probeAdvancedUI(probePage),
+                  ]),
+                  muTimeout,
+                ]);
+
+                const muPayload: Record<string, unknown> = { screenId: screen.id, url };
+                if (dropEnumRes.status === 'fulfilled' && (dropEnumRes.value as any).totalOptionsFound > 0)
+                  muPayload.dropdowns = dropEnumRes.value;
+                if (crudRes.status === 'fulfilled' && ((crudRes.value as any).flows?.length > 0 || (crudRes.value as any).toastsObserved?.length > 0))
+                  muPayload.crudFlows = crudRes.value;
+                if (filterRes.status === 'fulfilled' && (filterRes.value as any).filters?.length > 0)
+                  muPayload.filtersAndPagination = filterRes.value;
+                if (advancedRes.status === 'fulfilled' && advancedRes.value &&
+                    ((advancedRes.value as any).datePickers?.length > 0 || (advancedRes.value as any).sliders?.length > 0 ||
+                     (advancedRes.value as any).typeaheads?.length > 0 || (advancedRes.value as any).dragDrop?.library !== 'none'))
+                  muPayload.advancedUI = advancedRes.value;
+                if (Object.keys(muPayload).length > 2) {
+                  AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                    jobId: opts.jobId, type: 'PER_SCREEN_PROBES_MUTATING', payload: muPayload,
+                  }).catch(() => {});
+                }
+              } catch {
+                // non-fatal — probe failures must not block BFS
+              } finally {
+                await probePage?.close().catch(() => {});
+              }
+            })();
+          }
+        }
+
         // Capture DOM snapshot for structural analysis, and extract Markdown/text
         // content from the same DOM (crawl4AI-parity text output) — no extra fetch.
         try {
@@ -2769,10 +3424,38 @@ export async function crawlProject(opts: {
             try {
               const baseline = await AppBrain.getVisualBaselineForDiff(opts.tenantId, screen.id, 'DESKTOP', gitInfo.branch);
               if (baseline && baseline.storagePath !== shotPath) {
+                // Resolve baseline file path — local copy may have been rotated; fall back to cloud download
+                let baselinePath: string = baseline.storagePath;
+                let tempBaselinePath: string | null = null;
+                const localExists = await fs.access(baseline.storagePath).then(() => true, () => false);
+                if (!localExists) {
+                  if (baseline.cloudUrl) {
+                    try {
+                      const resp = await fetch(baseline.cloudUrl);
+                      if (resp.ok) {
+                        const buf = Buffer.from(await resp.arrayBuffer());
+                        tempBaselinePath = `${shotPath}.baseline-tmp.png`;
+                        await fs.writeFile(tempBaselinePath, buf);
+                        baselinePath = tempBaselinePath;
+                        logger.info({ screenId: screen.id }, '[crawler] VisualDiff baseline downloaded from cloud storage');
+                      } else {
+                        logger.warn({ screenId: screen.id, status: resp.status }, '[crawler] VisualDiff skipped — baseline cloud fetch returned non-200');
+                        return;
+                      }
+                    } catch (dlErr) {
+                      logger.warn({ screenId: screen.id, err: String(dlErr) }, '[crawler] VisualDiff skipped — baseline cloud download failed');
+                      return;
+                    }
+                  } else {
+                    logger.warn({ screenId: screen.id }, '[crawler] VisualDiff skipped — baseline not on disk and no cloudUrl available');
+                    return;
+                  }
+                }
+                try {
                 const { screenshotDiffPercent, gridLayoutDiff } = await import('./phash.js');
                 const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
-                  screenshotDiffPercent(baseline.storagePath, shotPath, baseline.ignoredRegions),
-                  gridLayoutDiff(baseline.storagePath, shotPath, baseline.ignoredRegions),
+                  screenshotDiffPercent(baselinePath, shotPath, baseline.ignoredRegions),
+                  gridLayoutDiff(baselinePath, shotPath, baseline.ignoredRegions),
                   AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, 'DESKTOP'),
                 ]);
                 let verdict = pHashVerdict(pHashDistance, changeLevel, [
@@ -2793,10 +3476,10 @@ export async function crawlProject(opts: {
                     const [baselineBuffer, newBuffer] = regions.length > 0
                       ? await (async () => {
                         const { applyMask } = await import('./phash.js');
-                        return Promise.all([applyMask(baseline.storagePath, regions), applyMask(shotPath, regions)]);
+                        return Promise.all([applyMask(baselinePath, regions), applyMask(shotPath, regions)]);
                       })()
                       : await Promise.all([
-                        fs.readFile(baseline.storagePath),
+                        fs.readFile(baselinePath),
                         fs.readFile(shotPath),
                       ]);
                     const aiResult = await aiVisualCompare({
@@ -2839,7 +3522,7 @@ export async function crawlProject(opts: {
                       // R5: Generate and upload pixelmatch diff image (changed pixels in red)
                       try {
                         const { generatePixelmatchDiff } = await import('./phash.js');
-                        const diffBuffer = await generatePixelmatchDiff(baseline.storagePath, shotPath, baseline.ignoredRegions ?? []);
+                        const diffBuffer = await generatePixelmatchDiff(baselinePath, shotPath, baseline.ignoredRegions ?? []);
                         const diffTmpPath = `${shotPath}.diff.png`;
                         await fs.writeFile(diffTmpPath, diffBuffer);
                         const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
@@ -2863,19 +3546,45 @@ export async function crawlProject(opts: {
                     aiSummary,
                   });
                 }
+                } finally {
+                  if (tempBaselinePath) fs.unlink(tempBaselinePath).catch(() => {});
+                }
               }
             } catch (err: any) {
-              const missing = String(err?.message ?? err).includes('Input file is missing');
-              if (missing) {
-                logger.warn({ screenId: screen.id, version: screen.version, err: String(err?.message ?? err) }, '[crawler] VisualDiff skipped — baseline screenshot not on disk (stored in cloud only). Run a full recrawl to regenerate baselines.');
-              } else {
-                console.warn('[crawler] VisualDiff pipeline failed:', err);
-              }
+              console.warn('[crawler] VisualDiff pipeline failed:', err);
             }
           })();
         }
 
         const linkMappings = await extractLinkMappings(page, origin);
+
+        // Auth-wall guard: check if the page drifted to a login/error URL during the async
+        // element extraction window (30-90s). If so, the `elements` array contains login-page
+        // elements — saving them would corrupt the screen's element catalog.
+        const postExtractUrl = (() => { try { return page.url(); } catch { return url; } })();
+        const postExtractParsed = (() => { try { return new URL(postExtractUrl); } catch { return new URL(url); } })();
+        const driftedToAuthWall = !weWantedThisPage && (
+          BFS_LOGIN_PATH.test(postExtractParsed.pathname) || BFS_ERROR_PATH.test(postExtractParsed.pathname)
+        );
+        if (driftedToAuthWall) {
+          logger.warn({ url, postExtractUrl }, '[crawler] Page drifted to auth wall during element extraction — skipping element save to prevent login-page element contamination');
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            screenId: screen.id,
+            jobId: opts.jobId,
+            type: 'AUTH_FAILURE',
+            payload: { url, landed: postExtractUrl, reason: 'Session expired during element extraction. Re-capture session and re-crawl this page.' },
+          }).catch(() => {});
+        } else {
+        if (elements.length === 0) {
+          // Extraction returned nothing — keep existing elements rather than wiping them.
+          // Causes: page.evaluate failure, contentFallbackElements threw, SPA not yet hydrated.
+          logger.warn({ url, screenId: screen.id }, '[crawler] extractElements returned 0 — preserving existing elements to avoid data loss');
+          // Queue for re-visit with longer wait — SPA content may not have hydrated yet
+          if (!revisitQueue.includes(url)) {
+            revisitQueue.push(url);
+            logger.info({ url }, '[crawler] 0-element page queued for re-visit with extended hydration wait');
+          }
+        } else {
         // Clear existing elements before re-saving — prevents duplicates on re-crawl
         await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
         // Batch element saves: 4 concurrent DB writes instead of serial round-trips
@@ -2895,6 +3604,7 @@ export async function crawlProject(opts: {
             AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, (saved as any).id, selectors).catch(console.warn);
           }));
         }
+        } // end inner else: elements.length > 0
         for (const lm of linkMappings) {
           const safety = assessActionSafety(lm.label, domainProfile);
           if (!safety.allowed) {
@@ -3056,6 +3766,7 @@ export async function crawlProject(opts: {
             } catch { /* non-fatal — Skyvern unavailable */ }
           }
         }
+        } // end auth-wall guard else
 
         if (rawPerf) {
           performanceMetrics.push({ url, screenName: name, ...rawPerf });
@@ -3169,16 +3880,16 @@ export async function crawlProject(opts: {
           if (visited.size >= maxScreens) break;
           try {
             await page.goto(skyvernUrl, { waitUntil: 'domcontentloaded', timeout: 20000 });
-            await page.waitForTimeout(1500);
+            await new Promise<void>(r => setTimeout(r, 1500));
             const skyvernName = new URL(skyvernUrl).pathname.split('/').filter(Boolean).pop() ?? 'Screen';
             const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, skyvernName, skyvernUrl, opts.jobId, 'SKYVERN_AI');
             const shotPath = `${opts.screenshotDir ?? '/tmp'}/${screen.id}-v${screen.version}.png`;
-            await waitForScreenshotReady(page);
+            await waitForScreenshotReady(page, rawPage ?? undefined);
             await page.screenshot({ path: shotPath, fullPage: true });
             await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP');
             const [elems, links, linkMappings] = await Promise.all([
-              extractElements(stagehand, page, elementLLM ?? undefined),
-              discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0),
+              extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
+              discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined),
               extractLinkMappings(page, origin),
             ]);
             for (const el of elems) {
@@ -3209,15 +3920,15 @@ export async function crawlProject(opts: {
           if (!isUsefulNavigationUrl(href, origin)) continue;
           try {
             await page.goto(href, { waitUntil: 'domcontentloaded', timeout: 20000 });
-            await page.waitForTimeout(1000);
+            await new Promise<void>(r => setTimeout(r, 1000));
             const nm = new URL(href).pathname.split('/').filter(Boolean).pop() ?? 'Screen';
             const sc = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, 'SKYVERN_AI');
             const sp = `${opts.screenshotDir ?? '/tmp'}/${sc.id}-v${sc.version}.png`;
-            await waitForScreenshotReady(page);
+            await waitForScreenshotReady(page, rawPage ?? undefined);
             await page.screenshot({ path: sp, fullPage: true });
             await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, 'DESKTOP');
             const [elems2, linkMappings2] = await Promise.all([
-              extractElements(stagehand, page, elementLLM ?? undefined),
+              extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
               extractLinkMappings(page, origin),
             ]);
             for (const el of elems2) {
@@ -3265,7 +3976,9 @@ export async function crawlProject(opts: {
         );
       }
     } catch (err) {
-      logger.warn({ err: String(err) }, '[P1] URL pattern completion failed — non-fatal, continuing');
+      const msg = String(err);
+      logger.warn({ err: msg }, '[P1] URL pattern completion failed — non-fatal, continuing');
+      crawlWarnings.push(`url-completer skipped: ${msg.includes('401') || msg.toLowerCase().includes('api key') || msg.toLowerCase().includes('authentication') ? 'LLM API key invalid or missing' : msg}`);
     }
 
     // P3a: Async content re-visit — revisit pages where DOM grew significantly during extraction.
@@ -3274,23 +3987,52 @@ export async function crawlProject(opts: {
       await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content…`);
       logger.info({ count: revisitQueue.length }, '[P3a] Starting async content re-visit pass');
       for (const revisitUrl of revisitQueue.slice(0, 10)) { // cap at 10 re-visits
-        const existingScreenId = visited.get(canonicalize(revisitUrl));
-        if (!existingScreenId) continue;
+        const rawScreenId = visited.get(canonicalize(revisitUrl));
+        // Auth-wall pages are stored as '_auth_wall_' — not a real screen ID.
+        // Navigate with the authenticated main page; it will redirect to the real content.
+        const isAuthWallRevisit = rawScreenId === '_auth_wall_';
+        if (!rawScreenId) continue;
         try {
           await page.goto(revisitUrl, { waitUntil: 'networkidle', timeout: 25_000 });
-          await page.waitForTimeout(2_000);
+          await new Promise<void>(r => setTimeout(r, 2_000));
+          await waitForScreenshotReady(page, rawPage ?? undefined);
+
+          // After navigation, check where we actually landed (auth may redirect)
+          const landedUrl = (rawPage ?? page).url?.() ?? revisitUrl;
+          const landedCanon = canonicalize(landedUrl);
+          let existingScreenId = isAuthWallRevisit
+            ? (visited.get(landedCanon) ?? null)  // use screen at redirected URL if known
+            : rawScreenId;
+
+          // If auth-wall redirect landed on an unvisited page, save it as a new screen
+          if (isAuthWallRevisit && (!existingScreenId || existingScreenId.startsWith('_')) && !visited.has(landedCanon)) {
+            const pathParts = new URL(landedUrl).pathname.split('/').filter(Boolean);
+            const screenName = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
+            const newScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, landedUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
+            if (newScreen) {
+              visited.set(landedCanon, newScreen.id);
+              existingScreenId = newScreen.id;
+            }
+          }
+
+          if (!existingScreenId || existingScreenId.startsWith('_')) {
+            logger.warn({ url: revisitUrl, landed: landedUrl }, '[P3a] Re-visit skipped — could not resolve screen ID');
+            continue;
+          }
+
           const shotPath2 = `${opts.screenshotDir ?? '/tmp'}/${existingScreenId}-revisit.png`;
-          await waitForScreenshotReady(page);
           await page.screenshot({ path: shotPath2, fullPage: true });
-          // Update screenshot with the more complete render
-          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, 'DESKTOP').catch(() => { });
+          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, 'DESKTOP', undefined).catch(() => {});
           // Re-extract elements with the async-loaded content now visible
-          const revisitElements = await extractElements(stagehand, page, elementLLM ?? undefined).catch(() => []);
-          for (const el of revisitElements) {
-            await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
+          const revisitElements = await extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []);
+          if (revisitElements.length > 0) {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, existingScreenId).catch(() => {});
+            for (const el of revisitElements) {
+              await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
+            }
           }
           // Harvest any new links revealed by async content
-          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false).catch(() => []);
+          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []);
           for (const href of revisitLinks) {
             const canon = canonicalize(href);
             if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
@@ -3298,13 +4040,66 @@ export async function crawlProject(opts: {
               queue.push(href);
             }
           }
-          logger.info({ url: revisitUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, '[P3a] Re-visit complete');
+          logger.info({ url: revisitUrl, landed: landedUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, '[P3a] Re-visit complete');
         } catch (err) {
           logger.warn({ url: revisitUrl, err: String(err) }, '[P3a] Re-visit failed — non-fatal');
         }
       }
     }
 
+    // P3b: Extended BFS — process any links discovered during P3a revisit pass.
+    // Without this, auth-wall pages redirect to real app pages, AI discovers new links,
+    // but those links are pushed to queue and NEVER processed (main BFS already exited).
+    if (queue.length > 0 && visited.size < maxScreens) {
+      const p3bCount = Math.min(queue.length, maxScreens - visited.size);
+      logger.info({ newLinks: p3bCount }, '[P3b] Extended BFS — processing links discovered during revisit pass');
+      await onProgress?.(82, `Extended crawl: ${p3bCount} newly discovered page(s)…`);
+      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        const href = queue.shift();
+        if (!href) break;
+        const canonHref = canonicalize(href);
+        if (visited.has(canonHref)) continue;
+        if (!isUsefulNavigationUrl(href, origin)) continue;
+        visited.set(canonHref, '_pending_');
+        try {
+          await page.goto(href, { waitUntil: 'networkidle', timeout: 25_000 }).catch(() => {});
+          await waitForScreenshotReady(page, rawPage ?? undefined);
+          const pathParts = new URL(href).pathname.split('/').filter(Boolean);
+          const nm = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, 'LINK_FOLLOW');
+          visited.set(canonHref, screen.id);
+          const sp = `${opts.screenshotDir ?? '/tmp'}/${screen.id}-v${screen.version}.png`;
+          await page.screenshot({ path: sp, fullPage: true }).catch(() => {});
+          const cloudUrl3 = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => '');
+          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, sp, screen.version, 'DESKTOP', cloudUrl3).catch(() => {});
+          const [p3bElems, p3bLinks] = await Promise.all([
+            extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
+            discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined).catch(() => []),
+          ]);
+          if (p3bElems.length > 0) {
+            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
+            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 4) {
+              const _b3 = p3bElems.slice(_ei3, _ei3 + 4);
+              await Promise.all(_b3.map((el: any) =>
+                AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {})
+              ));
+            }
+          }
+          for (const link of p3bLinks) {
+            const c = canonicalize(link);
+            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 2) {
+              queued.add(c); queue.push(link);
+            }
+          }
+          discovered.push({ url: href, screenId: screen.id, elements: p3bElems.length, version: screen.version });
+          logger.info({ url: href, elements: p3bElems.length, newLinks: p3bLinks.length }, '[P3b] Extended BFS screen captured');
+        } catch (err) {
+          logger.warn({ url: href, err: String(err) }, '[P3b] Extended BFS URL failed');
+          visited.set(canonHref, '_error_');
+        }
+      }
+    }
+
     // P6: Gap Pass — LLM gap analysis on discovered screens to find structurally missing pages.
     // Runs after P3a re-visits so the gap analysis has the most complete screen list possible.
     // Confirmed gap URLs (HEAD=200) are visited directly with the existing browser.
@@ -3344,16 +4139,16 @@ export async function crawlProject(opts: {
                 logger.warn({ gapUrl, landed: page.url() }, '[P6] Gap URL redirected to login — skipping');
                 continue;
               }
-              await page.waitForTimeout(1_500);
+              await new Promise<void>(r => setTimeout(r, 1_500));
               const gapName = gap.category;
               const gapScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, gapName, gapUrl, opts.jobId, 'ENTRY_POINT');
               const gapShot = `${opts.screenshotDir ?? '/tmp'}/${gapScreen.id}-v${gapScreen.version}.png`;
-              await waitForScreenshotReady(page);
+              await waitForScreenshotReady(page, rawPage ?? undefined);
               await page.screenshot({ path: gapShot, fullPage: true });
               await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, 'DESKTOP');
               const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
-                extractElements(stagehand, page, elementLLM ?? undefined).catch(() => []),
-                discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false).catch(() => []),
+                extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
+                discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []),
                 extractLinkMappings(page, origin).catch(() => []),
               ]);
               for (const el of gapElems) {
@@ -3380,14 +4175,23 @@ export async function crawlProject(opts: {
           logger.info({ total: gapCandidates.length }, '[P6] Gap pass: no confirmed gaps found');
         }
       } catch (err) {
-        logger.warn({ err: String(err) }, '[P6] Gap pass failed — non-fatal, continuing');
+        const msg = String(err);
+        logger.warn({ err: msg }, '[P6] Gap pass failed — non-fatal, continuing');
+        crawlWarnings.push(`gap-analysis skipped: ${msg.includes('401') || msg.toLowerCase().includes('api key') || msg.toLowerCase().includes('authentication') ? 'LLM API key invalid or missing' : msg}`);
       }
     }
 
     // After crawl: mark screens not re-discovered as PENDING_REMOVAL (non-destructive diff)
     // Only run if crawl actually found screens — 0 discovered means failed/aborted run, not evidence of removal
     if (!selective && visited.size > 0) {
-      const discoveredUrls = Array.from(visited.keys());
+      // Only pass canonical URLs of screens that were actually visited (saveScreen called).
+      // Exclude marker strings (_skipped_api_, _pending_, _soft404_, pre-auth, skipped)
+      // so that skipped API/export paths don't pollute the discoveredSet and prevent removal
+      // of old API-path screens from previous crawls.
+      const VISIT_MARKER_RE = /^_|^pre-auth$|^skipped$/;
+      const discoveredUrls = Array.from(visited.entries())
+        .filter(([, v]) => typeof v === 'string' && !VISIT_MARKER_RE.test(v))
+        .map(([k]) => k);
       await AppBrain.markRemovedScreens(opts.tenantId, opts.projectId, discoveredUrls, opts.jobId);
     }
 
@@ -3579,6 +4383,7 @@ export async function crawlProject(opts: {
             ignoreHTTPSErrors: true,
             ...(mainStorageState ? { storageState: mainStorageState } : {}),
           });
+          mobileCtx.on('download', (dl: any) => { dl.cancel().catch(() => {}); });
           const mobilePage = await mobileCtx.newPage();
 
           for (const disc of discovered) {
@@ -3588,12 +4393,15 @@ export async function crawlProject(opts: {
               await mobilePage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
               await dismissOverlays(mobilePage);
               try { await mobilePage.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' }); } catch { /* ignore */ }
+              // mobilePage is a real Playwright page — pass it as rawPage so CDP wheel events
+              // trigger virtual scrollers (react-window / TanStack Virtual) at mobile viewports.
+              await infiniteScrollToBottom(mobilePage, { maxScrolls: 20, scrollDelay: 300, stabilizeMs: 400 }, mobilePage).catch(() => { });
               const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${disc.screenId}-${profile.toLowerCase()}-v${disc.version}.png`);
-              await waitForScreenshotReady(mobilePage);
+              await waitForScreenshotReady(mobilePage, mobilePage);
               // Screenshot and DOM element extraction run in parallel
               const [, vpElements] = await Promise.all([
                 mobilePage.screenshot({ path: shotPath, fullPage: true }),
-                domFallbackElements(mobilePage),
+                domFallbackElements(mobilePage, mobilePage),
               ]);
               const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
               await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version!, profile, cloudUrl);
@@ -3638,6 +4446,26 @@ export async function crawlProject(opts: {
       await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
     }
 
+    // Step 7: End-of-crawl state graph coverage report.
+    // Per-screen probes (dropdown, CRUD, filter, DnD, nested-contexts, shadow-DOM,
+    // accessibility-tree, performance, service-worker) run per-screen inside the BFS
+    // loop above and are saved as PER_SCREEN_PROBES evidence.
+    try {
+      await onProgress?.(90, 'Generating state graph coverage report');
+      const stateGraph = new ScreenTransitionGraph();
+      const coveragePayload: Record<string, unknown> = {
+        stateGraph: stateGraph.getCoverageReport(),
+        totalScreensCrawled: discovered.filter(d => !!d.screenId).length,
+      };
+      AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+        jobId: opts.jobId,
+        type: 'CRAWL_COVERAGE_REPORT',
+        payload: coveragePayload,
+      }).catch(() => { });
+    } catch (extErr) {
+      logger.warn({ err: String(extErr) }, '[coverage-report] non-fatal error');
+    }
+
     await onProgress?.(97, 'Running post-crawl analysis');
 
     // ── Per-crawl summaries (fire-and-forget, non-blocking) ────────────────
@@ -3714,6 +4542,17 @@ export async function crawlProject(opts: {
     const screensSucceeded = discovered.filter(d => !!d.screenId).length;
     const screensErrored = discovered.filter(d => !!d.error && !d.screenId).length;
 
+    // Detect systematic element/link extraction failure across all screens
+    if (screensSucceeded >= 3) {
+      const screensWithElements = discovered.filter(d => (d.elements ?? 0) > 0 && d.screenId).length;
+      if (screensWithElements === 0) {
+        crawlWarnings.push(
+          `element-extraction: 0 elements found across all ${screensSucceeded} screens — ` +
+          `DOM evaluate may be failing (check server logs for "page.evaluate threw" warnings)`,
+        );
+      }
+    }
+
     return {
       screensFound: screensSucceeded,           // successful captures only (not errors)
       screensErrored,                           // pages visited but failed to capture
@@ -3726,6 +4565,7 @@ export async function crawlProject(opts: {
       segmentIndex: segIdx,                     // which segment this was
       hasMoreSegments,
       domHashes: Array.from(seenDomHashes),     // carry forward to next segment to avoid re-screenshotting
+      crawlWarnings: crawlWarnings.length > 0 ? crawlWarnings : undefined,
     };
   } finally {
     // Log accumulated Stagehand LLM usage (extract/act/observe calls) to cost ledger.
@@ -3760,11 +4600,13 @@ export async function crawlProject(opts: {
 // Elements whose meanings indicate UI state toggles or noise — filtered from both DOM and AI results.
 const ELEMENT_NOISE = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
 
-async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
+async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, rawPage?: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
   // llmCfg reserved for per-task model selection (future: create task-scoped Stagehand)
   // DOM-first: fast, no LLM cost, covers server-rendered and most SPAs.
   // Threshold >= 2 so a lone dark-mode button doesn't suppress AI fallback for rich pages.
-  const domElements = await domFallbackElements(page).catch((err: any) => {
+  // rawPage: real Playwright CDP page used for page.evaluate() — avoids Stagehand's proxy
+  // re-bundling the function with esbuild (which injects __name, breaking browser eval).
+  const domElements = await domFallbackElements(page, rawPage).catch((err: any) => {
     console.warn('[crawler] domFallbackElements failed (page context issue or DOM error):', err?.message ?? err);
     return [] as Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>;
   });
@@ -3811,7 +4653,10 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
   // Always capture content elements (headings, meta, cards, nav, FAQ) regardless of interactive count.
   // Previously this was fallback-only; now both paths run together so static sections are
   // always visible to the test generator even on pages with forms.
-  const contentElements = await contentFallbackElements(page);
+  const contentElements = await contentFallbackElements(page).catch((err: any) => {
+    console.warn('[crawler] contentFallbackElements failed (page context issue):', err?.message ?? err);
+    return [] as Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number> }>;
+  });
 
   // Cross-source text deduplication: strips wrapper phrases like "Button to expand '...' details"
   // and "... FAQ button" so duplicate FAQ buttons from DOM + content sources are collapsed.
@@ -3867,14 +4712,14 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
   }
 }
 
-async function discoverLinks(stagehand: Stagehand, origin: string, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, skipAi = false): Promise<string[]> {
+async function discoverLinks(stagehand: Stagehand, origin: string, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, skipAi = false, rawPage?: any): Promise<string[]> {
   // Expand collapsed nav menus (hamburger, aria-haspopup, Radix closed triggers) so hidden routes
   // become part of the DOM before querying. This catches SPA nav dropdowns that only render on click.
   const navExpanded = await expandNavMenus(page);
 
   // DOM-first: <a href> links — standard apps. AI supplements when DOM coverage is < 20 unique links.
   // Threshold of 20 (not 8) because nav sidebars quickly provide 8+ links while missing deep content URLs.
-  const domLinks = await domFallbackLinks(page, origin);
+  const domLinks = await domFallbackLinks(page, origin, rawPage);
 
   // Interact with tabs, carousels, and accordions to reveal hidden content and discover more links.
   const interactiveLinks = await interactAndCollectLinks(page, origin);
@@ -3929,8 +4774,9 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
   }
 }
 
-export async function domFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
-  return page.evaluate(() => {
+export async function domFallbackElements(page: any, rawPage?: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
+  // Named function so we can .toString() it for CDP Runtime.evaluate fallback
+  function _browserEval() {
     const out: { meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }[] = [];
     const seen = new Set<string>();
     // UI chrome noise: cookie banners, theme toggles, version strings, social/share buttons
@@ -3949,8 +4795,8 @@ export async function domFallbackElements(page: any): Promise<Array<{ meaning: s
       return results;
     };
 
-    // Expanded selector — all interactive ARIA roles + native form elements + rich editors
-    const selector = 'input:not([type=hidden]),select,textarea,button,[role="button"],[role="checkbox"],[role="radio"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],[role="gridcell"],[role="link"],a[role="button"],[contenteditable="true"]';
+    // Expanded selector — all interactive ARIA roles + native form elements + rich editors + bare nav links
+    const selector = 'input:not([type=hidden]),select,textarea,button,a[href],[role="button"],[role="checkbox"],[role="radio"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],[role="gridcell"],[role="link"],[contenteditable="true"]';
 
     querySelectorAllDeep(document, selector).forEach((el: any) => {
       const style = window.getComputedStyle(el);
@@ -3969,14 +4815,13 @@ export async function domFallbackElements(page: any): Promise<Array<{ meaning: s
       const ariaRole = el.getAttribute('role');
       const isActionEl = tag === 'button' || tag === 'a' || ariaRole === 'button';
 
-      // Skip plain anchor links inside <header> or <footer> — navigation chrome with no test value.
+      // Skip plain anchor links inside <footer> only — footer chrome (sitemap, legal links) has no test value.
+      // Header nav links ARE test targets (SPA primary navigation lives in <header>) — do NOT skip.
       // Do NOT skip <nav> elements: SPA dashboard sidebars live in <nav> and are primary test targets.
-      // Buttons inside nav (e.g. collapse, settings) are kept; only <a> links are skipped.
-      if (tag === 'a' || (ariaRole === 'button' && tag !== 'button')) {
+      if (tag === 'a') {
         let p = el.parentElement;
         while (p) {
-          const ptag = p.tagName?.toLowerCase();
-          if (['header', 'footer'].includes(ptag)) return;
+          if (p.tagName?.toLowerCase() === 'footer') return;
           p = p.parentElement;
         }
       }
@@ -4157,7 +5002,64 @@ export async function domFallbackElements(page: any): Promise<Array<{ meaning: s
       ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea', 'url', 'file'].includes(e.role) ? 0 : 1;
     out.sort((a, b) => priority(a) - priority(b));
     return out; // Cap removed
-  }).then((elems: any[]) => elems.map((e: any) => ({ ...e, isInteractive: true }))).catch(() => []);
+  }
+  // CDP Runtime.evaluate bypasses Playwright's isolated world entirely.
+  // The "__name is not defined" error comes from Playwright's own UtilityScript being absent
+  // from the isolated world (destroyed by navigation events in Stagehand's CDP session).
+  // Solution: when rawPage (a real Playwright Page) is available, use CDP Runtime.evaluate
+  // as the PRIMARY path — no isolated world needed, no __name dependency.
+  //
+  // esbuild injects __name() calls for named arrow-function constants inside _browserEval
+  // (e.g. querySelectorAllDeep). When _browserEval.toString() is sent to the browser,
+  // the browser throws "ReferenceError: __name is not defined". Fix: prepend a tiny
+  // __name polyfill that just returns the function (name-setting is non-critical).
+  const _browserEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_browserEval.toString()})()})();`;
+
+  if (rawPage && typeof rawPage.context === 'function') {
+    try {
+      const ctx = rawPage.context();
+      if (typeof ctx?.newCDPSession === 'function') {
+        const cdp = await ctx.newCDPSession(rawPage);
+        try {
+          const evalResult = await cdp.send('Runtime.evaluate', {
+            expression: _browserEvalStr,
+            returnByValue: true,
+            awaitPromise: false,
+          });
+          if (evalResult?.exceptionDetails) {
+            console.warn('[crawler] domFallbackElements CDP primary error:', evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
+          } else if (Array.isArray(evalResult?.result?.value)) {
+            return (evalResult.result.value as any[]).map((e: any) => ({ ...e, isInteractive: true }));
+          }
+        } finally {
+          cdp.detach().catch(() => {});
+        }
+      }
+    } catch (cdpPrimaryErr: any) {
+      console.warn('[crawler] domFallbackElements CDP primary failed, trying page.evaluate:', cdpPrimaryErr?.message ?? String(cdpPrimaryErr));
+    }
+  }
+
+  // Fallback: pass the pre-serialized string expression so esbuild's __name() wrapper
+  // for nested named functions never reaches the browser (Function.prototype.toString
+  // returns the source including __name() calls for any nested named arrow constants).
+  // Playwright accepts a JS expression string directly — no isolated-world __name leakage.
+  const evalPage = rawPage ?? page;
+  const _doEvaluate = (): Promise<any> => evalPage.evaluate(_browserEvalStr as any);
+  let result: any[];
+  try {
+    result = await _doEvaluate();
+  } catch (err1: any) {
+    // Second attempt after a brief wait — isolated world may need a tick to settle
+    await new Promise<void>(r => setTimeout(r, 500));
+    try {
+      result = await _doEvaluate();
+    } catch (err2: any) {
+      console.warn('[crawler] domFallbackElements page.evaluate threw (attempt 2):', err2?.message ?? String(err2));
+      return [];
+    }
+  }
+  return (result ?? []).map((e: any) => ({ ...e, isInteractive: true }));
 }
 
 /** Capture content anatomy for EVERY page — not just static ones.
@@ -4277,19 +5179,28 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
       const s = window.getComputedStyle(el);
       return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
     };
-    document.querySelectorAll('nav a, [role="navigation"] a, header a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a: any) => {
+    const navSel = [
+      'nav a', '[role="navigation"] a', 'header a', 'header nav a', 'footer nav a',
+      '[aria-label*="navigation" i] a', '[aria-label*="menu" i] a', '[aria-label*="sidebar" i] a',
+      'aside a', '[role="complementary"] a',
+      '[class*="sidebar" i] a', '[class*="side-nav" i] a', '[class*="sidenav" i] a',
+      '[class*="nav-menu" i] a', '[class*="navmenu" i] a', '[data-nav] a',
+    ].join(', ');
+    const navLinksFoundCount = { n: 0 };
+    document.querySelectorAll(navSel).forEach((a: any) => {
       if (!isNavVisible(a)) return;
       const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
       const href = a.href;
       if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
-      // Walk up to find nearest nav/header/footer ancestor to get landmark label
       let landmark = 'Site Navigation';
       let el: any = a.parentElement;
       while (el) {
         const tag = el.tagName?.toLowerCase();
         const role = el.getAttribute?.('role')?.toLowerCase();
+        const cl = (el.className ?? '').toLowerCase();
         if (tag === 'header' || role === 'banner') { landmark = 'Header'; break; }
         if (tag === 'footer' || role === 'contentinfo') { landmark = 'Footer'; break; }
+        if (tag === 'aside' || role === 'complementary' || cl.includes('sidebar') || cl.includes('sidenav') || cl.includes('side-nav')) { landmark = 'Sidebar'; break; }
         if (tag === 'nav' || role === 'navigation') {
           const directLabel = el.getAttribute('aria-label')?.trim() ?? '';
           const labelId = el.getAttribute('aria-labelledby');
@@ -4300,7 +5211,43 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
         el = el.parentElement;
       }
       addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark }) });
+      navLinksFoundCount.n++;
     });
+    // Grandparent-grouping fallback: detects sidebar navigation where each link is individually
+    // wrapped (e.g. <div class="flex"><a>link</a></div> × many siblings). Runs ALWAYS so header
+    // links don't block sidebar detection. Deduped against hrefs already in the semantic pass.
+    {
+      const navHrefsAlready = new Set<string>();
+      for (const item of out) {
+        if (item.role === 'nav-link') {
+          try { const h = JSON.parse(item.notes ?? '{}').href; if (h) navHrefsAlready.add(h); } catch { /* skip */ }
+        }
+      }
+      const gpGroups2 = new Map<Element, Array<{label: string; href: string}>>();
+      document.querySelectorAll('a[href]').forEach((a: any) => {
+        if (!isNavVisible(a)) return;
+        const href = a.href ?? '';
+        if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsAlready.has(href)) return;
+        const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+        if (!label || label.length > 40) return;
+        const parent = a.parentElement;
+        const gp = parent?.parentElement;
+        if (!parent || !gp) return;
+        if (parent.querySelectorAll('a').length !== 1) return;
+        const gpTag = gp.tagName.toLowerCase();
+        if (['body', 'html', 'form'].includes(gpTag)) return;
+        const arr = gpGroups2.get(gp) ?? [];
+        arr.push({ label, href });
+        gpGroups2.set(gp, arr);
+      });
+      for (const [, links] of gpGroups2.entries()) {
+        if (links.length >= 4) {
+          for (const { label, href } of links) {
+            addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark: 'Sidebar' }) });
+          }
+        }
+      }
+    }
 
     // 9. Breadcrumb items
     document.querySelectorAll('[aria-label*="breadcrumb" i] a, [class*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] a').forEach((a: any) => {
@@ -5029,15 +5976,12 @@ async function expandNavMenus(page: any): Promise<boolean> {
   } catch { return false; }
 }
 
-async function domFallbackLinks(page: any, origin: string): Promise<string[]> {
-  return page.evaluate((origin: string) => {
+async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promise<string[]> {
+  // Named function so .toString() gives the raw function body for CDP evaluation.
+  // esbuild wraps nested named arrow-function constants with __name() — prepend a
+  // tiny polyfill so the browser has it when we serialize this function as a string.
+  function _linkEval(originStr: string) {
     const logout = /logout|log out|sign out|delete/i;
-    // Hash-routed SPAs (React Router hash history, Vue Router hash mode, old
-    // AngularJS) encode the actual route as the fragment — "#/dashboard",
-    // "#!/settings" — as distinct from a same-page scroll anchor like "#top"
-    // or "#section-2". Treating every hash as a scroll anchor (stripping it
-    // entirely) collapses an entire hash-routed app down to its single base
-    // URL, discovering only ~1 screen no matter how many routes it has.
     const isRouteHash = (hash: string) => /^#!?\//.test(hash);
     const seen = new Set<string>();
     const add = (rawHref: string, textContent: string) => {
@@ -5048,11 +5992,9 @@ async function domFallbackLinks(page: any, origin: string): Promise<string[]> {
       try {
         const u = new URL(rawHref, location.href);
         if (!isRouteHash(u.hash)) u.hash = '';
-        if (new URL(u).origin === origin) seen.add(u.toString());
+        if (new URL(u).origin === originStr) seen.add(u.toString());
       } catch { /* ignore */ }
     };
-
-    // Iterative BFS shadow DOM traversal — avoids stack overflow on deeply nested shadow roots
     const allRoots: Array<Document | ShadowRoot> = [document];
     const rootSeen = new WeakSet<Document | ShadowRoot>();
     rootSeen.add(document);
@@ -5073,12 +6015,48 @@ async function domFallbackLinks(page: any, origin: string): Promise<string[]> {
               rootSeen.add(el.shadowRoot);
               allRoots.push(el.shadowRoot);
             }
-          } catch (_) { /* closed/cross-origin shadow root — skip */ }
+          } catch (_) { /* closed/cross-origin shadow root */ }
         });
-      } catch (_) { /* element threw — skip this root */ }
+      } catch (_) { /* element threw — skip root */ }
     }
     return Array.from(seen);
-  }, origin).catch(() => []);
+  }
+
+  const _linkEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_linkEval.toString()})(${JSON.stringify(origin)})})();`;
+
+  // CDP primary — bypasses Stagehand isolated world entirely
+  if (rawPage && typeof rawPage.context === 'function') {
+    try {
+      const ctx = rawPage.context();
+      if (typeof ctx?.newCDPSession === 'function') {
+        const cdp = await ctx.newCDPSession(rawPage);
+        try {
+          const evalResult = await cdp.send('Runtime.evaluate', {
+            expression: _linkEvalStr,
+            returnByValue: true,
+            awaitPromise: false,
+          });
+          if (!evalResult?.exceptionDetails && Array.isArray(evalResult?.result?.value)) {
+            return evalResult.result.value as string[];
+          }
+          if (evalResult?.exceptionDetails) {
+            console.warn('[crawler] domFallbackLinks CDP error:', evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
+          }
+        } finally {
+          cdp.detach().catch(() => {});
+        }
+      }
+    } catch (cdpErr: any) {
+      console.warn('[crawler] domFallbackLinks CDP failed:', cdpErr?.message ?? String(cdpErr));
+    }
+  }
+
+  // Fallback: pass string expression so esbuild __name wrapper never reaches browser
+  const evalPage = rawPage ?? page;
+  return (evalPage.evaluate(_linkEvalStr as any) as Promise<string[]>).catch((err: any) => {
+    console.warn('[crawler] domFallbackLinks page.evaluate threw:', err?.message ?? String(err));
+    return [];
+  });
 }
 
 /** Extract label→href pairs for all in-app anchor links — used to seed KAG element transitions. */
@@ -5190,57 +6168,122 @@ async function dismissOverlays(page: any): Promise<void> {
       }
     } catch { /* overlay not present — skip */ }
   }
-  // Brief settle after dismissals
-  await page.waitForTimeout(300).catch(() => { });
+  // Brief settle after dismissals — Node.js timer, no dependency on page.waitForTimeout proxy
+  await new Promise<void>(r => setTimeout(r, 300));
 }
 
 /**
  * Scroll the full page height to trigger IntersectionObserver-based lazy loads
  * (images, components, infinite-scroll content). Scrolls back to top before capture.
  */
-async function scrollToTriggerLazyLoad(page: any): Promise<void> {
+async function scrollToTriggerLazyLoad(page: any, rawPage?: any): Promise<void> {
+  const evalPage = rawPage ?? page;
   try {
-    await page.evaluate(async () => {
-      const step = Math.max(window.innerHeight, 400);
-      const total = Math.min(document.body.scrollHeight, 12_000); // cap at 12k px to avoid infinite pages
-      for (let y = step; y < total; y += step) {
-        window.scrollTo(0, y);
-        await new Promise<void>(r => setTimeout(r, 120));
+    // Scroll the full page in viewport-sized steps.
+    // window.scrollTo handles standard scroll containers and triggers IntersectionObserver.
+    const { scrollHeight, innerHeight, innerWidth } = await evalPage.evaluate(() => ({
+      scrollHeight: Math.min(document.body.scrollHeight, 12_000),
+      innerHeight: window.innerHeight || 800,
+      innerWidth: window.innerWidth || 1280,
+    })).catch(() => ({ scrollHeight: 0, innerHeight: 800, innerWidth: 1280 }));
+
+    if (scrollHeight > innerHeight) {
+      // CDP wheel events trigger virtual scrollers (react-window / TanStack Virtual / AG Grid)
+      // which ignore window.scrollTo but listen on their own container via wheel events.
+      let cdp: any = null;
+      if (rawPage && typeof rawPage.context === 'function') {
+        try {
+          const ctx = rawPage.context();
+          if (typeof ctx?.newCDPSession === 'function') cdp = await ctx.newCDPSession(rawPage);
+        } catch { cdp = null; }
       }
-      window.scrollTo(0, 0);
-    });
+
+      const step = Math.max(innerHeight, 400);
+      for (let y = step; y < scrollHeight; y += step) {
+        await evalPage.evaluate((pos: number) => window.scrollTo(0, pos), y).catch(() => {});
+        if (cdp) {
+          await cdp.send('Input.dispatchMouseEvent', {
+            type: 'mouseWheel',
+            x: Math.floor(innerWidth / 2),
+            y: Math.floor(innerHeight / 2),
+            deltaX: 0,
+            deltaY: step,
+            modifiers: 0,
+          }).catch(() => {});
+        }
+        // Node.js-side wait — does NOT depend on page.waitForTimeout (Stagehand proxy may lack it)
+        await new Promise<void>(r => setTimeout(r, 150));
+      }
+      // Scroll back to top
+      await evalPage.evaluate(() => window.scrollTo(0, 0)).catch(() => {});
+      if (cdp) cdp.detach().catch(() => {});
+    }
+
     // Let lazy-triggered content render before extraction
-    await page.waitForTimeout(400).catch(() => { });
-    // Re-check DOM stability after scroll-triggered mutations
-    await waitForStableDOM(page, 3_000, 300);
+    await new Promise<void>(r => setTimeout(r, 400));
+    // Wait for networkidle: IntersectionObserver fires on scroll but async data fetches
+    // (React Query, SWR, Apollo) triggered by it may still be in-flight. networkidle ensures
+    // all XHR/fetch requests spawned by the scroll complete before DOM stability check.
+    const scrollWaitPage = rawPage ?? page;
+    if (typeof scrollWaitPage.waitForLoadState === 'function') {
+      await scrollWaitPage.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
+    }
+    // Re-check DOM stability (shadow-piercing) after scroll-triggered mutations + data fetches
+    await waitForStableDOM(page, 4_000, 500, rawPage);
   } catch { /* scroll failed — non-fatal, proceed */ }
 }
 
-async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400): Promise<void> {
+async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400, rawPage?: any): Promise<void> {
+  const evalPage = rawPage ?? page; // prefer rawPage — avoids Stagehand proxy + __name isolated-world issues
   const deadline = Date.now() + maxWait;
   let lastCount = -1;
   let stableAt = 0;
+  let evalErrors = 0;
   while (Date.now() < deadline) {
-    const count: number = await page.evaluate(() => document.querySelectorAll('*').length).catch(() => 0);
-    if (count !== lastCount) {
-      lastCount = count;
-      stableAt = Date.now();
-    } else if (Date.now() - stableAt >= stabilityMs) {
-      return;
+    const count: number = await evalPage.evaluate(() => {
+      // Shadow-piercing counter: querySelectorAll('*') only counts light-DOM elements;
+      // shadow roots are separate trees. Counting both ensures changes inside web components
+      // (e.g. Material UI, Lit, Shoelace) reset the stability timer correctly.
+      let c = 0;
+      const visit = (root: Document | ShadowRoot) => {
+        const els = root.querySelectorAll('*');
+        c += els.length;
+        els.forEach((el: any) => { if (el.shadowRoot) visit(el.shadowRoot); });
+      };
+      visit(document);
+      return c;
+    }).catch(() => -1);
+    if (count === -1) {
+      // evaluate failed (isolated world broken) — after 3 consecutive failures, wait stabilityMs and bail
+      if (++evalErrors >= 3) {
+        await new Promise(r => setTimeout(r, stabilityMs));
+        return;
+      }
+    } else {
+      evalErrors = 0;
+      if (count !== lastCount) {
+        lastCount = count;
+        stableAt = Date.now();
+      } else if (Date.now() - stableAt >= stabilityMs) {
+        return;
+      }
     }
     await new Promise(r => setTimeout(r, 100));
   }
   // Timed out — DOM may still be loading, proceed anyway
 }
 
-async function waitForScreenshotReady(page: any): Promise<void> {
-  const hardDeadline = Date.now() + 20_000;
+async function waitForScreenshotReady(page: any, rawPage?: any): Promise<void> {
+  const hardDeadline = Date.now() + 30_000;
   const withHardCap = <T>(p: Promise<T>): Promise<T | void> =>
     Promise.race([p, new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
+  // Use rawPage for load-state waits — Stagehand proxy may not expose waitForLoadState
+  const waitPage = rawPage ?? page;
+  const evalPage = rawPage ?? page;
   try {
-    await withHardCap(page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { }));
-    await withHardCap(page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { }));
-    await withHardCap(page.evaluate(() => {
+    await withHardCap(waitPage.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { }));
+    await withHardCap(waitPage.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { }));
+    await withHardCap(evalPage.evaluate(() => {
       if (document.readyState === 'complete') return true;
       return new Promise<boolean>((resolve) => {
         const done = () => resolve(true);
@@ -5248,13 +6291,13 @@ async function waitForScreenshotReady(page: any): Promise<void> {
         setTimeout(done, 4_000);
       });
     }).catch(() => { }));
-    await withHardCap(page.evaluate(async () => {
+    await withHardCap(evalPage.evaluate(async () => {
       const fonts = (document as any).fonts;
       if (fonts?.ready) await fonts.ready;
     }).catch(() => { }));
-    await withHardCap(scrollToTriggerLazyLoad(page));
-    await withHardCap(waitForStableDOM(page, 4_000, 700));
-    await withHardCap(page.evaluate(async () => {
+    await withHardCap(scrollToTriggerLazyLoad(rawPage ?? page, rawPage));
+    await withHardCap(waitForStableDOM(page, 4_000, 700, rawPage));
+    await withHardCap(evalPage.evaluate(async () => {
       const pending = Array.from(document.images).filter((img) => !img.complete);
       if (pending.length === 0) return;
       await Promise.race([
@@ -5265,7 +6308,7 @@ async function waitForScreenshotReady(page: any): Promise<void> {
         new Promise<void>((resolve) => setTimeout(resolve, 4_000)),
       ]);
     }).catch(() => { }));
-    await withHardCap(waitForStableDOM(page, 2_000, 500));
+    await withHardCap(waitForStableDOM(page, 2_000, 500, rawPage));
   } catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
 }
 
@@ -5490,6 +6533,9 @@ async function runAuthDiscoveryPhase(
 
   while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
     const current = queue.shift()!;
+    const _authPct = 10 + Math.round((captured / AUTH_DISCOVERY_MAX_SCREENS) * 7);
+    const _authPath = current.url.replace(/^https?:\/\/[^/]+/, '') || '/';
+    await onProgress?.(_authPct, `Auth discovery · ${_authPath} · ${captured} found`);
     try {
       const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).catch(() => null);
       if (response && response.status && response.status() >= 400) continue;