@detiq/crawler 1.1.19 → 1.1.21

package/src/crawler.ts

@@ -257,31 +257,67 @@ function normalizeAuthUrl(url: string): string {
 }
 
 async function capturePageHtml(page: any): Promise<string | null> {
+  let html: string | null = null;
+
   try {
     if (typeof page.content === 'function') {
-      const html = await page.content();
-      if (typeof html === 'string' && html.trim()) return html;
+      const raw = await page.content();
+      if (typeof raw === 'string' && raw.trim()) html = raw;
     }
   } catch (err: any) {
     logger.warn({ err: String(err?.message ?? err) }, '[crawler] page.content() failed; trying DOM evaluate fallback');
   }
 
+  if (!html) {
+    try {
+      html = await page.evaluate(() => {
+        const type = document.doctype;
+        const doctype = type
+          ? '<!DOCTYPE ' + type.name +
+            (type.publicId ? ' PUBLIC "' + type.publicId + '"' : '') +
+            (type.systemId ? ' "' + type.systemId + '"' : '') +
+            '>'
+          : '<!DOCTYPE html>';
+        return doctype + '\n' + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? '');
+      });
+      if (typeof html !== 'string' || !html.trim()) html = null;
+    } catch (err: any) {
+      logger.warn({ err: String(err?.message ?? err) }, '[crawler] DOM snapshot capture failed');
+      return null;
+    }
+  }
+
+  if (!html) return null;
+
+  // Inline all accessible stylesheets so the snapshot renders correctly offline
+  // (srcDoc iframes have a null origin — external CSS loads fail cross-origin).
   try {
-    const html = await page.evaluate(() => {
-      const type = document.doctype;
-      const doctype = type
-        ? '<!DOCTYPE ' + type.name +
-          (type.publicId ? ' PUBLIC "' + type.publicId + '"' : '') +
-          (type.systemId ? ' "' + type.systemId + '"' : '') +
-          '>'
-        : '<!DOCTYPE html>';
-      return doctype + '\n' + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? '');
+    html = await page.evaluate(() => {
+      // Strip script tags — execution is blocked by CSP/sandbox anyway; removing reduces size and avoids leaking inline tokens.
+      document.querySelectorAll('script').forEach((s: any) => s.remove());
+      for (const sheet of Array.from(document.styleSheets)) {
+        try {
+          // cssRules throws SecurityError for cross-origin sheets — skip those
+          const cssText = Array.from(sheet.cssRules).map((r: any) => r.cssText).join('\n');
+          const style = document.createElement('style');
+          style.textContent = cssText;
+          const owner = sheet.ownerNode as Element | null;
+          owner?.replaceWith(style);
+        } catch { /* cross-origin CDN sheet — leave <link> in place */ }
+      }
+      const doctype = (() => {
+        const dt = document.doctype;
+        return dt ? `<!DOCTYPE ${dt.name}>` : '<!DOCTYPE html>';
+      })();
+      return doctype + '\n' + (document.documentElement?.outerHTML ?? '');
     });
-    return typeof html === 'string' && html.trim() ? html : null;
+    if (typeof html !== 'string' || !html.trim()) html = null;
   } catch (err: any) {
-    logger.warn({ err: String(err?.message ?? err) }, '[crawler] DOM snapshot capture failed');
-    return null;
+    logger.warn({ err: String(err?.message ?? err) }, '[crawler] CSS inlining failed; using raw HTML');
+    // html already has the raw value from page.content() — usable as-is
   }
+
+  return html;
 }
 
 /** Launch Chrome on a random CDP port and return its WebSocket debugger URL. */
@@ -442,6 +478,10 @@ const TwoFASchema = z.object({
   fieldLabel: z.string().optional(),
 });
 
+// Shared across all crawl phases (lite-worker, main BFS, P3a, P6).
+// PHP/legacy apps render session-expired content inline without URL redirect.
+const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
+
 export async function crawlProject(opts: {
   tenantId: string;
   projectId: string;
@@ -1086,6 +1126,14 @@ export async function crawlProject(opts: {
               if (popupScreen) {
                 const cloudUrl = await uploadToCloud(popupShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(popupShotPath)}`).catch(() => undefined);
                 await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, popupScreen.id, popupShotPath, popupScreen.version, 'DESKTOP', cloudUrl).catch(() => {});
+                // GAP 4: DOM snapshot for popup capture
+                const popupDom = await capturePageHtml(popup);
+                if (popupDom) {
+                  const popupDomPath = popupShotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+                  await fs.writeFile(popupDomPath, popupDom, 'utf8');
+                  const popupDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(popupDomPath)}`;
+                  AppBrain.updateScreenDomSnapshot(opts.tenantId, popupScreen.id, popupDomPath, await uploadToCloud(popupDomPath, popupDomKey).catch(() => undefined)).catch(() => {});
+                }
                 const popupEls = await domFallbackElements(popup, popup).catch(() => []);
                 for (const el of popupEls) {
                   AppBrain.saveElement(opts.tenantId, opts.projectId, popupScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {});
@@ -1267,7 +1315,7 @@ export async function crawlProject(opts: {
             } catch { return ''; }
           })
           .filter((h: string, i: number, arr: string[]) => h && arr.indexOf(h) === i)
-          .slice(0, 30);
+          .slice(0, 50);
       }, _appOrigin).catch(() => [] as string[]);
 
       for (const link of publicNavLinks) {
@@ -1307,6 +1355,14 @@ export async function crawlProject(opts: {
           await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {});
           const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
           await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl).catch(() => {});
+          // GAP 5: DOM snapshot for pre-auth public page discovery
+          const pubDom = await capturePageHtml(page);
+          if (pubDom) {
+            const pubDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+            await fs.writeFile(pubDomPath, pubDom, 'utf8');
+            const pubDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(pubDomPath)}`;
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, pubDomPath, await uploadToCloud(pubDomPath, pubDomKey).catch(() => undefined)).catch(() => {});
+          }
           // Save elements found in the pre-auth view
           await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
           for (const el of preAuthElements) {
@@ -1687,6 +1743,10 @@ export async function crawlProject(opts: {
     let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
     let lastEmittedPct = 20; // floor above auth-discovery + login phases (max ~19%)
     const crawlLoopStart = Date.now();
+    // Global crawl timeout — prevent BFS from hanging indefinitely on large apps.
+    // Default 2 hours; override via CRAWL_GLOBAL_TIMEOUT_MS env var.
+    const CRAWL_GLOBAL_TIMEOUT_MS = parseInt(process.env.CRAWL_GLOBAL_TIMEOUT_MS ?? '') || 2 * 60 * 60 * 1000;
+    const isCrawlTimedOut = () => (Date.now() - crawlLoopStart) >= CRAWL_GLOBAL_TIMEOUT_MS;
     const startUrl = page.url();
     // Resume from previous segment's pending queue if provided
     const resumeUrls = opts.resumeUrls ?? [];
@@ -1726,6 +1786,11 @@ export async function crawlProject(opts: {
         ];
     const queued = new Set<string>(seedUrls.map(canonicalize));
     const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
+    // Selective recrawl: block all link-discovery pushes so BFS only visits the requested URLs.
+    // Constructor seeding already ran above — this no-op override fires only for BFS-loop discoveries.
+    if (selective) {
+      queue.push = () => {};
+    }
     const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? '') || 0.88);
     const deadLinkTracker = new DeadLinkTracker();
     const redirectRegistry = new RedirectRegistry();
@@ -1824,7 +1889,8 @@ export async function crawlProject(opts: {
           logger.info({ count: g11SitemapUrls.length }, '[crawler] G11: sitemap discovered public URL candidates');
           const sitemapOrigin = new URL(opts.appUrl).origin;
           let sitemapCaptured = 0;
-          for (const sUrl of g11SitemapUrls.slice(0, 30)) {
+          for (const sUrl of g11SitemapUrls) {
+            if (isCrawlTimedOut()) break;
             try {
               const u = new URL(sUrl);
               if (u.origin !== sitemapOrigin) continue;
@@ -1846,7 +1912,17 @@ export async function crawlProject(opts: {
               await page.screenshot({ path: shotPath, fullPage: true }).catch(() => { });
               const pubScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, pageName, sUrl, opts.jobId, 'SITEMAP').catch(() => null);
               if (pubScreen) {
-                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, 'DESKTOP').catch(() => { });
+                const sitemapCloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, 'DESKTOP', sitemapCloudUrl).catch(() => { });
+                // GAP 2: DOM snapshot for G11 pre-auth sitemap path
+                const sitemapDom = await capturePageHtml(page);
+                if (sitemapDom) {
+                  const sitemapDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+                  await fs.writeFile(sitemapDomPath, sitemapDom, 'utf8');
+                  const sitemapDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(sitemapDomPath)}`;
+                  const sitemapDomCloud = await uploadToCloud(sitemapDomPath, sitemapDomKey).catch(() => undefined);
+                  AppBrain.updateScreenDomSnapshot(opts.tenantId, pubScreen.id, sitemapDomPath, sitemapDomCloud).catch(() => {});
+                }
                 logger.info({ url: sUrl, name: pageName }, '[crawler] G11: captured public sitemap page');
               }
               sitemapCaptured++;
@@ -1928,6 +2004,7 @@ export async function crawlProject(opts: {
     async function runLiteWorker(workerPage: any): Promise<void> {
       while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
         if (opts.abortSignal?.aborted) break;
+        if (isCrawlTimedOut()) break;
         if (++totalAttemptsThisSegment > maxAttempts) break;
 
         const url = queue.shift();
@@ -1999,6 +2076,18 @@ export async function crawlProject(opts: {
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
           visited.set(canonUrl, screen.id);
 
+          // Inline auth-wall detection: PHP/legacy apps render session-expired content without URL redirect.
+          // URL-redirect case (lwIsAuthWall above) is handled by main BFS via revisitQueue.
+
+          const lwBodyText = await workerPage.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
+          if (INLINE_AUTH_WALL_RE.test(lwBodyText)) {
+            logger.warn({ url, screenId: screen.id }, '[lite-worker] Inline session-expired content detected — marking requiresAuth');
+            AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {});
+            visited.set(canonUrl, '_auth_wall_');
+            if (!revisitQueue.includes(url)) revisitQueue.push(url);
+            continue;
+          }
+
           const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
 
           let liteElements = await domFallbackElements(workerPage).catch(() => [] as Awaited<ReturnType<typeof domFallbackElements>>);
@@ -2026,6 +2115,16 @@ export async function crawlProject(opts: {
               .catch(() => {}),
           ]);
 
+          // GAP 1: DOM snapshot for lite-worker path
+          const lwDom = await capturePageHtml(workerPage);
+          if (lwDom) {
+            const lwDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+            await fs.writeFile(lwDomPath, lwDom, 'utf8');
+            const lwDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(lwDomPath)}`;
+            const lwDomCloud = await uploadToCloud(lwDomPath, lwDomKey).catch(() => undefined);
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, lwDomPath, lwDomCloud).catch(() => {});
+          }
+
           // Nav-link pass — merged into main save cycle so clearScreenElements covers them too.
           // extractElements (main BFS) never runs for lite-worker screens, so nav/footer roles
           // must be captured here. Running before clear+save prevents additive accumulation.
@@ -2122,8 +2221,8 @@ export async function crawlProject(opts: {
           // Persist lite-worker elements to DB — without this, 80% of screens have 0 DB elements.
           if (allElements.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
-            for (let _ei = 0; _ei < allElements.length; _ei += 4) {
-              const _batch = allElements.slice(_ei, _ei + 4);
+            for (let _ei = 0; _ei < allElements.length; _ei += 2) {
+              const _batch = allElements.slice(_ei, _ei + 2);
               await Promise.all(_batch.map((el: any) =>
                 AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {})
               ));
@@ -2235,6 +2334,12 @@ export async function crawlProject(opts: {
         logger.info('Abort signal received — stopping crawl');
         throw new Error('CRAWL_ABORTED');
       }
+      if (isCrawlTimedOut()) {
+        const elapsedMin = Math.round((Date.now() - crawlLoopStart) / 60_000);
+        logger.warn({ elapsedMin, visited: visited.size, queued: queue.length }, '[crawler] Global timeout reached — stopping BFS to prevent runaway crawl');
+        await onProgress?.(89, `Crawl timeout after ${elapsedMin}min — ${visited.size} pages captured`).catch(() => {});
+        break;
+      }
       // AutoThrottle: delay next request proportional to avg navigation latency
       if (opts.autoThrottle && navCount > 0) {
         const delayMs = Math.min(5_000, Math.max(0, avgNavMs * 0.5));
@@ -2475,7 +2580,17 @@ export async function crawlProject(opts: {
               }).catch(() => 'Modal');
               const modalScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, modalName, url, opts.jobId, discoveryMethodFor(url)).catch(() => null);
               if (modalScreen) {
-                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, 'DESKTOP').catch(() => { });
+                const modalCloudUrl = await uploadToCloud(modalShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(modalShotPath)}`).catch(() => undefined);
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, 'DESKTOP', modalCloudUrl).catch(() => { });
+                // GAP 3: DOM snapshot for modal/dialog capture
+                const modalDom = await capturePageHtml(page);
+                if (modalDom) {
+                  const modalDomPath = modalShotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+                  await fs.writeFile(modalDomPath, modalDom, 'utf8');
+                  const modalDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(modalDomPath)}`;
+                  const modalDomCloud = await uploadToCloud(modalDomPath, modalDomKey).catch(() => undefined);
+                  AppBrain.updateScreenDomSnapshot(opts.tenantId, modalScreen.id, modalDomPath, modalDomCloud).catch(() => {});
+                }
                 // Extract elements from the modal BEFORE it is dismissed — use CDP path via rawPage
                 const modalElements = await domFallbackElements(page, rawPage ?? undefined).catch(() => []);
                 for (const el of modalElements) {
@@ -2531,6 +2646,11 @@ export async function crawlProject(opts: {
                   reason: `Login wall detected: ${loginWallRedirectCount} consecutive URLs redirected to login. Add credentials in project settings to crawl authenticated pages.`,
                 },
               }).catch(() => {});
+              // Checkpoint remaining queue so job-queue can auto-requeue the pending URLs
+              await opts.onCheckpoint?.(
+                Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
+                queue.filter((u: string) => !visited.has(canonicalize(u)))
+              ).catch(() => {});
               throw new Error(`[AUTH_FAILURE] Login wall detected: ${loginWallRedirectCount} URLs redirected to the login page. Add credentials in project settings and retry the crawl.`);
             }
           }
@@ -2562,6 +2682,11 @@ export async function crawlProject(opts: {
                 type: 'AUTH_FAILURE',
                 payload: { url, landed: landedUrl, reason: 'Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.' },
               }).catch(() => { });
+              // Checkpoint remaining queue before aborting so job-queue can auto-requeue
+              await opts.onCheckpoint?.(
+                Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
+                queue.filter((u: string) => !visited.has(canonicalize(u)))
+              ).catch(() => {});
               throw new Error('[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.');
             }
           } else {
@@ -2578,6 +2703,11 @@ export async function crawlProject(opts: {
               type: 'AUTH_FAILURE',
               payload: { url, landed: landedUrl, reason: 'Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.' },
             }).catch(() => { });
+            // Checkpoint remaining queue before aborting so job-queue can auto-requeue
+            await opts.onCheckpoint?.(
+              Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
+              queue.filter((u: string) => !visited.has(canonicalize(u)))
+            ).catch(() => {});
             throw new Error('[AUTH_FAILURE] Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.');
           }
         }
@@ -3256,6 +3386,20 @@ export async function crawlProject(opts: {
           }
         }
 
+        // Inline auth-wall detection: PHP/legacy apps render session-expired content
+        // without changing the URL, bypassing all URL-path-based guards above.
+        const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
+        const pageBodyText = await page.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
+        if (INLINE_AUTH_WALL_RE.test(pageBodyText)) {
+          logger.warn({ url, screenId: screen.id }, '[crawler] Inline session-expired content detected — skipping DOM snapshot, marking requiresAuth');
+          AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {});
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            screenId: screen.id, jobId: opts.jobId,
+            type: 'AUTH_FAILURE',
+            payload: { url, reason: 'Inline session-expired page rendered without URL redirect. Re-authenticate and re-crawl.' },
+          }).catch(() => {});
+        } else {
+
         // Capture DOM snapshot for structural analysis, and extract Markdown/text
         // content from the same DOM (crawl4AI-parity text output) — no extra fetch.
         try {
@@ -3265,7 +3409,12 @@ export async function crawlProject(opts: {
           } else {
             const domPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
             await fs.writeFile(domPath, domContent, 'utf8');
-            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath).catch(() => { });
+            // Upload DOM snapshot to cloud alongside PNG, then persist both paths
+            (async () => {
+              const domKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(domPath)}`;
+              const domCloudUrl = await uploadToCloud(domPath, domKey).catch(() => undefined);
+              AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath, domCloudUrl).catch(() => {});
+            })();
 
             // Supplement markdown with iFrame and Shadow DOM content
             const [iframeContent, shadowContent] = await Promise.all([
@@ -3374,6 +3523,7 @@ export async function crawlProject(opts: {
         } catch (err: any) {
           logger.warn({ url, screenId: screen.id, err: String(err?.message ?? err) }, '[crawler] markdown extraction failed');
         }
+        } // end: inline auth-wall else
 
         // Persist a11y score + individual violations — fire-and-forget, non-fatal
         if (a11yScore !== null) {
@@ -3568,6 +3718,7 @@ export async function crawlProject(opts: {
         );
         if (driftedToAuthWall) {
           logger.warn({ url, postExtractUrl }, '[crawler] Page drifted to auth wall during element extraction — skipping element save to prevent login-page element contamination');
+          AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {});
           AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
             screenId: screen.id,
             jobId: opts.jobId,
@@ -3587,9 +3738,9 @@ export async function crawlProject(opts: {
         } else {
         // Clear existing elements before re-saving — prevents duplicates on re-crawl
         await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
-        // Batch element saves: 4 concurrent DB writes instead of serial round-trips
-        for (let _ei = 0; _ei < elements.length; _ei += 4) {
-          const _batch = elements.slice(_ei, _ei + 4);
+        // Batch element saves: 2 concurrent DB writes (4 caused pool saturation at 5 workers)
+        for (let _ei = 0; _ei < elements.length; _ei += 2) {
+          const _batch = elements.slice(_ei, _ei + 2);
           await Promise.all(_batch.map(async (el: any) => {
             const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark);
             // Populate ElementFingerprint — fire-and-forget, failures are non-fatal
@@ -3601,7 +3752,7 @@ export async function crawlProject(opts: {
               { type: 'aria-label', value: label, confidence: 0.85 },
               { type: 'tag-hint', value: selectorHint, confidence: 0.6 },
             ];
-            AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, (saved as any).id, selectors).catch(console.warn);
+            await AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, (saved as any).id, selectors).catch(console.warn);
           }));
         }
         } // end inner else: elements.length > 0
@@ -3634,7 +3785,7 @@ export async function crawlProject(opts: {
                 isUsefulNavigationUrl(lm.href, origin) &&
                 !queued.has(canonLmHref) &&
                 !visited.has(canonLmHref) &&
-                queued.size < maxScreens * 2
+                queued.size < maxScreens * 3
               ) {
                 if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
                 queued.add(canonLmHref);
@@ -3658,7 +3809,7 @@ export async function crawlProject(opts: {
               if (/^yith-|^wc-|^vc_|^elementor/.test(actionVal)) continue;
             } catch { /* invalid URL, let it fall through */ }
             const canonHref = canonicalize(href);
-            if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 2) {
+            if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 3) {
               // Don't flood the queue with generated ID-like route segments.
               try {
                 const hrefPath = new URL(href).pathname;
@@ -3717,7 +3868,7 @@ export async function crawlProject(opts: {
               const jsDetectedRoutes = await detectJsRoutes(page, origin);
               for (const route of jsDetectedRoutes) {
                 const canonRoute = canonicalize(route);
-                if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 2) {
+                if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 3) {
                   queued.add(canonRoute);
                   queue.push(route, { parentElements: elements.length });
                   logger.info({ route }, '[P2] JS router: discovered route added to queue');
@@ -3733,7 +3884,7 @@ export async function crawlProject(opts: {
             try {
               const abs = `${origin}${relPath}`;
               const canon = canonicalize(abs);
-              if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+              if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
                 queued.add(canon);
                 queue.push(abs);
                 logger.info({ url: abs }, '[P4] XHR: page URL from JSON response added to queue');
@@ -3745,7 +3896,7 @@ export async function crawlProject(opts: {
           // Triggered per-page when: zero DOM links found + fewer than 5 elements (dead-end / canvas page).
           // Skyvern returns URLs it discovered → added to queue for ZeTa's own BFS crawl (not saved directly).
           // ZeTa does all actual crawling — Skyvern is only a URL-discovery override layer.
-          if (nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? '') || 5) && opts.skyvernConfig) {
+          if (!selective && nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? '') || 5) && opts.skyvernConfig) {
             try {
               logger.info({ url }, 'Gap 5: thin page detected — asking Skyvern to discover navigation URLs');
               const skyvernNav = await skyvernNavigate(
@@ -3755,7 +3906,7 @@ export async function crawlProject(opts: {
               );
               for (const skyvernUrl of skyvernNav.visitedUrls) {
                 const canonSkyvernUrl = canonicalize(skyvernUrl);
-                if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 2) {
+                if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 3) {
                   logger.info({ url: skyvernUrl }, 'Gap 5: Skyvern-discovered URL added to ZeTa crawl queue');
                   queued.add(canonSkyvernUrl);
                   queue.push(skyvernUrl); // ZeTa will crawl this normally
@@ -3821,9 +3972,9 @@ export async function crawlProject(opts: {
         await onProgress?.(clampedPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
         logger.info({ name, elements: elements.length }, 'Screen crawled');
         // Checkpoint every 10 successes so a failed job can be resumed from close to where it stopped
-        if (++checkpointSuccessCount % 10 === 0) {
+        if (++checkpointSuccessCount % 5 === 0) {
           await opts.onCheckpoint?.(
-            Array.from(visited.keys()),
+            Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
             queue.filter(u => !visited.has(canonicalize(u)))
           );
         }
@@ -3861,7 +4012,7 @@ export async function crawlProject(opts: {
     // Use only successfully-saved screens (screenId present), not total discovered including errors.
     const successfulScreenCount = discovered.filter(d => d.screenId).length;
     const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? '') || 10;
-    if (successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
+    if (!selective && successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
       await onProgress?.(78, 'Few screens found — using Skyvern to discover additional navigation URLs');
       const currentUrl = page.url();
       const skyvernNav = await skyvernNavigate(
@@ -3877,7 +4028,7 @@ export async function crawlProject(opts: {
         logger.info({ count: skyvernUrls.length }, 'Gap 5: Skyvern found new URLs — processing with ZeTa native crawler');
         // Process each URL with ZeTa's own Playwright-based extraction
         for (const skyvernUrl of skyvernUrls) {
-          if (visited.size >= maxScreens) break;
+          if (isCrawlTimedOut() || visited.size >= maxScreens) break;
           try {
             await page.goto(skyvernUrl, { waitUntil: 'domcontentloaded', timeout: 20000 });
             await new Promise<void>(r => setTimeout(r, 1500));
@@ -3886,7 +4037,17 @@ export async function crawlProject(opts: {
             const shotPath = `${opts.screenshotDir ?? '/tmp'}/${screen.id}-v${screen.version}.png`;
             await waitForScreenshotReady(page, rawPage ?? undefined);
             await page.screenshot({ path: shotPath, fullPage: true });
-            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP');
+            const skyvernCloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', skyvernCloudUrl);
+            // GAP 7: DOM snapshot for Skyvern primary URLs
+            const skyvernDom = await capturePageHtml(page);
+            if (skyvernDom) {
+              const skyvernDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+              await fs.writeFile(skyvernDomPath, skyvernDom, 'utf8');
+              const skyvernDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(skyvernDomPath)}`;
+              const skyvernDomCloud = await uploadToCloud(skyvernDomPath, skyvernDomKey).catch(() => undefined);
+              AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, skyvernDomPath, skyvernDomCloud).catch(() => {});
+            }
             const [elems, links, linkMappings] = await Promise.all([
               extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
               discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined),
@@ -3913,7 +4074,7 @@ export async function crawlProject(opts: {
           }
         }
         // Drain any newly queued URLs from Skyvern pages
-        while (queue.length > 0 && visited.size < maxScreens) {
+        while (queue.length > 0 && visited.size < maxScreens && !isCrawlTimedOut()) {
           const href = queue.shift()!;
           const canonHref = canonicalize(href);
           if (visited.has(canonHref)) continue;
@@ -3926,7 +4087,17 @@ export async function crawlProject(opts: {
             const sp = `${opts.screenshotDir ?? '/tmp'}/${sc.id}-v${sc.version}.png`;
             await waitForScreenshotReady(page, rawPage ?? undefined);
             await page.screenshot({ path: sp, fullPage: true });
-            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, 'DESKTOP');
+            const skyvernDrainCloudUrl = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => undefined);
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, 'DESKTOP', skyvernDrainCloudUrl);
+            // GAP 8: DOM snapshot for Skyvern drain-queue URLs
+            const skyvernDrainDom = await capturePageHtml(page);
+            if (skyvernDrainDom) {
+              const skyvernDrainDomPath = sp.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+              await fs.writeFile(skyvernDrainDomPath, skyvernDrainDom, 'utf8');
+              const skyvernDrainDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(skyvernDrainDomPath)}`;
+              const skyvernDrainDomCloud = await uploadToCloud(skyvernDrainDomPath, skyvernDrainDomKey).catch(() => undefined);
+              AppBrain.updateScreenDomSnapshot(opts.tenantId, sc.id, skyvernDrainDomPath, skyvernDrainDomCloud).catch(() => {});
+            }
             const [elems2, linkMappings2] = await Promise.all([
               extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
               extractLinkMappings(page, origin),
@@ -3986,7 +4157,8 @@ export async function crawlProject(opts: {
     if (revisitQueue.length > 0) {
       await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content…`);
       logger.info({ count: revisitQueue.length }, '[P3a] Starting async content re-visit pass');
-      for (const revisitUrl of revisitQueue.slice(0, 10)) { // cap at 10 re-visits
+      for (const revisitUrl of revisitQueue.slice(0, 20)) { // cap at 20 re-visits; also guard global timeout
+        if (isCrawlTimedOut()) break;
         const rawScreenId = visited.get(canonicalize(revisitUrl));
         // Auth-wall pages are stored as '_auth_wall_' — not a real screen ID.
         // Navigate with the authenticated main page; it will redirect to the real content.
@@ -4022,7 +4194,17 @@ export async function crawlProject(opts: {
 
           const shotPath2 = `${opts.screenshotDir ?? '/tmp'}/${existingScreenId}-revisit.png`;
           await page.screenshot({ path: shotPath2, fullPage: true });
-          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, 'DESKTOP', undefined).catch(() => {});
+          const p3aCloudUrl = await uploadToCloud(shotPath2, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath2)}`).catch(() => undefined);
+          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, 'DESKTOP', p3aCloudUrl).catch(() => {});
+          // GAP 10: DOM snapshot for P3a auth-wall revisit
+          const p3aDom = await capturePageHtml(page);
+          if (p3aDom) {
+            const p3aDomPath = shotPath2.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+            await fs.writeFile(p3aDomPath, p3aDom, 'utf8');
+            const p3aDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(p3aDomPath)}`;
+            const p3aDomCloud = await uploadToCloud(p3aDomPath, p3aDomKey).catch(() => undefined);
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, existingScreenId, p3aDomPath, p3aDomCloud).catch(() => {});
+          }
           // Re-extract elements with the async-loaded content now visible
           const revisitElements = await extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []);
           if (revisitElements.length > 0) {
@@ -4030,12 +4212,19 @@ export async function crawlProject(opts: {
             for (const el of revisitElements) {
               await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
             }
+          } else {
+            // P3a got 0 elements — check if it's an inline session-expired page.
+            const p3aBodyText = await page.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
+            if (INLINE_AUTH_WALL_RE.test(p3aBodyText)) {
+              logger.warn({ url: revisitUrl, screenId: existingScreenId }, '[P3a] Re-visit got 0 elements — inline session-expired content, marking requiresAuth');
+              AppBrain.updateScreenRequiresAuth(opts.tenantId, existingScreenId, true).catch(() => {});
+            }
           }
           // Harvest any new links revealed by async content
           const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []);
           for (const href of revisitLinks) {
             const canon = canonicalize(href);
-            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
               queued.add(canon);
               queue.push(href);
             }
@@ -4055,6 +4244,7 @@ export async function crawlProject(opts: {
       logger.info({ newLinks: p3bCount }, '[P3b] Extended BFS — processing links discovered during revisit pass');
       await onProgress?.(82, `Extended crawl: ${p3bCount} newly discovered page(s)…`);
       while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        if (isCrawlTimedOut()) break;
         const href = queue.shift();
         if (!href) break;
         const canonHref = canonicalize(href);
@@ -4072,14 +4262,23 @@ export async function crawlProject(opts: {
           await page.screenshot({ path: sp, fullPage: true }).catch(() => {});
           const cloudUrl3 = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => '');
           await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, sp, screen.version, 'DESKTOP', cloudUrl3).catch(() => {});
+          // GAP 6: DOM snapshot for P3b extended BFS
+          const p3bDom = await capturePageHtml(page);
+          if (p3bDom) {
+            const p3bDomPath = sp.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+            await fs.writeFile(p3bDomPath, p3bDom, 'utf8');
+            const p3bDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(p3bDomPath)}`;
+            const p3bDomCloud = await uploadToCloud(p3bDomPath, p3bDomKey).catch(() => undefined);
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, p3bDomPath, p3bDomCloud).catch(() => {});
+          }
           const [p3bElems, p3bLinks] = await Promise.all([
             extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
             discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined).catch(() => []),
           ]);
           if (p3bElems.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
-            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 4) {
-              const _b3 = p3bElems.slice(_ei3, _ei3 + 4);
+            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 2) {
+              const _b3 = p3bElems.slice(_ei3, _ei3 + 2);
               await Promise.all(_b3.map((el: any) =>
                 AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {})
               ));
@@ -4087,7 +4286,7 @@ export async function crawlProject(opts: {
           }
           for (const link of p3bLinks) {
             const c = canonicalize(link);
-            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 2) {
+            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 3) {
               queued.add(c); queue.push(link);
             }
           }
@@ -4129,9 +4328,9 @@ export async function crawlProject(opts: {
 
           // Visit each confirmed gap URL with the existing browser (depth=1, no link traversal)
           const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
-          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? '') || 8)) {
+          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? '') || 25)) {
+            if (isCrawlTimedOut() || visited.size >= maxScreens) break;
             const gapUrl = gap.confirmedUrl!;
-            if (visited.size >= maxScreens) break;
             try {
               await page.goto(gapUrl, { waitUntil: 'domcontentloaded', timeout: 20_000 });
               // Skip if session expired — don't save a login-page screenshot as the gap result
@@ -4145,7 +4344,17 @@ export async function crawlProject(opts: {
               const gapShot = `${opts.screenshotDir ?? '/tmp'}/${gapScreen.id}-v${gapScreen.version}.png`;
               await waitForScreenshotReady(page, rawPage ?? undefined);
               await page.screenshot({ path: gapShot, fullPage: true });
-              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, 'DESKTOP');
+              const gapCloudUrl = await uploadToCloud(gapShot, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(gapShot)}`).catch(() => undefined);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, 'DESKTOP', gapCloudUrl);
+              // GAP 9: DOM snapshot for P6 gap-pass
+              const gapDom = await capturePageHtml(page);
+              if (gapDom) {
+                const gapDomPath = gapShot.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+                await fs.writeFile(gapDomPath, gapDom, 'utf8');
+                const gapDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(gapDomPath)}`;
+                const gapDomCloud = await uploadToCloud(gapDomPath, gapDomKey).catch(() => undefined);
+                AppBrain.updateScreenDomSnapshot(opts.tenantId, gapScreen.id, gapDomPath, gapDomCloud).catch(() => {});
+              }
               const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
                 extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
                 discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []),
@@ -4154,6 +4363,13 @@ export async function crawlProject(opts: {
               for (const el of gapElems) {
                 await AppBrain.saveElement(opts.tenantId, opts.projectId, gapScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
               }
+              if (gapElems.length === 0) {
+                const gapBodyText = await page.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
+                if (INLINE_AUTH_WALL_RE.test(gapBodyText)) {
+                  logger.warn({ url: gapUrl, screenId: gapScreen.id }, '[P6] Gap URL got 0 elements — inline session-expired content, marking requiresAuth');
+                  AppBrain.updateScreenRequiresAuth(opts.tenantId, gapScreen.id, true).catch(() => {});
+                }
+              }
               visited.set(canonicalize(gapUrl), gapScreen.id);
               discovered.push({ url: gapUrl, screenId: gapScreen.id, elements: gapElems.length, version: gapScreen.version });
               for (const lm of gapLinkMaps) {
@@ -4162,7 +4378,7 @@ export async function crawlProject(opts: {
               // Enqueue gap page's links for further crawl (within maxScreens budget)
               for (const href of gapLinks) {
                 const canon = canonicalize(href);
-                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
                   queued.add(canon); queue.push(href);
                 }
               }
@@ -4387,6 +4603,7 @@ export async function crawlProject(opts: {
           const mobilePage = await mobileCtx.newPage();
 
           for (const disc of discovered) {
+            if (isCrawlTimedOut()) break;
             if (!disc.screenId) continue;
             try {
               await mobilePage.goto(disc.url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
@@ -4537,8 +4754,17 @@ export async function crawlProject(opts: {
 
     await onProgress?.(98, 'Finalizing');
     // Remaining queue = URLs discovered but not yet crawled due to segment limit
+    const wasTimeout = isCrawlTimedOut();
     const pendingUrls = queue.filter((u) => !visited.has(canonicalize(u)));
+    // All fully-processed URLs (exclude '_pending_' entries interrupted by timeout/abort)
+    const visitedUrls = Array.from(visited.entries())
+      .filter(([_k, v]) => v !== '_pending_')
+      .map(([k]) => k);
     const hasMoreSegments = pendingUrls.length > 0;
+    if (wasTimeout && pendingUrls.length > 0) {
+      const elapsedMin = Math.round((Date.now() - crawlLoopStart) / 60_000);
+      crawlWarnings.push(`global-timeout: crawl stopped after ${elapsedMin} minutes with ${pendingUrls.length} URLs remaining`);
+    }
     const screensSucceeded = discovered.filter(d => !!d.screenId).length;
     const screensErrored = discovered.filter(d => !!d.error && !d.screenId).length;
 
@@ -4562,6 +4788,8 @@ export async function crawlProject(opts: {
       apiEndpoints: capturedApiEndpoints,
       performanceMetrics,
       pendingUrls,                              // feed into next segment's resumeUrls
+      visitedUrls,                              // all fully-processed URLs for skipUrls in continuation
+      wasTimeout,                               // true when global timeout triggered the BFS break
       segmentIndex: segIdx,                     // which segment this was
       hasMoreSegments,
       domHashes: Array.from(seenDomHashes),     // carry forward to next segment to avoid re-screenshotting
@@ -4621,7 +4849,7 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
       if (!iframeUrl || iframeUrl === 'about:blank' || iframeUrl.startsWith('javascript:')) continue;
       const frameEls = await domFallbackElements(frame).catch(() => []);
       // Tag iframe elements with source context
-      for (const el of frameEls.slice(0, 10)) {
+      for (const el of frameEls.slice(0, 30)) {
         iframeElements.push({ ...el, notes: el.notes ? `${el.notes} [in iframe]` : '[in iframe]' });
       }
     }
@@ -4645,7 +4873,7 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
           seen.add(label);
           out.push({ meaning: label, role, notes: '[dynamic/revealed]' });
         });
-        return out.slice(0, 10);
+        return out.slice(0, 30);
       }).catch(() => []);
     } catch { dynamicElements = []; }
   }
@@ -4706,7 +4934,7 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
     if (!extracted) return domElements;
     // Apply noise filter to AI results — AI doesn't always follow skip instructions
     const filtered = (extracted as any).elements.filter((e: any) => !ELEMENT_NOISE.test(e.meaning ?? ''));
-    return filtered.slice(0, 20);
+    return filtered.slice(0, 40);
   } catch {
     return domElements;
   }
@@ -4737,9 +4965,9 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
 
   const allDomLinks = Array.from(new Set([...domLinks, ...interactiveLinks]));
 
-  if (allDomLinks.length >= 30 || skipAi) return allDomLinks;
+  if (allDomLinks.length >= 50 || skipAi) return allDomLinks;
 
-  // AI complement: DOM found fewer than 30 unique links — SPA likely has JS-router pages not in <a href>.
+  // AI complement: DOM found fewer than 50 unique links — SPA likely has JS-router pages not in <a href>.
   // Always merge DOM + AI so we never lose already-discovered links.
   // 12s timeout: allows 1 OpenAI retry (4.8s delay) but bails before the 2nd retry wastes more time.
   try {
@@ -5151,7 +5379,7 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     const mainContent = document.querySelector('main, [role="main"], article') ?? document.body;
     let listItemCount = 0;
     mainContent.querySelectorAll('li').forEach((li: any) => {
-      if (listItemCount >= 30) return;
+      if (listItemCount >= 50) return;
       if (!isVisible(li)) return;
       const text = li.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
       if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }, li); listItemCount++; }
@@ -5355,11 +5583,11 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
       if (url) { addUniq({ meaning: `Lazy BG image: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, pattern: 'data-bg' }) }); bgCount++; }
     });
 
-    // 12. Content links outside nav/header/footer (capped at 20)
+    // 12. Content links outside nav/header/footer (capped at 40)
     const navTags = new Set(['nav', 'header', 'footer']);
     let contentLinkCount = 0;
     document.querySelectorAll('a[href]').forEach((a: any) => {
-      if (contentLinkCount >= 20) return;
+      if (contentLinkCount >= 40) return;
       if (!isVisible(a)) return;
       let p = a.parentElement;
       while (p) { if (navTags.has(p.tagName?.toLowerCase())) return; p = p.parentElement; }
@@ -5848,7 +6076,7 @@ async function interactAndCollectLinks(page: any, origin: string): Promise<strin
       document.querySelectorAll('[role="tab"]').length
     ).catch(() => 0);
 
-    for (let i = 0; i < Math.min(tabCount, 15); i++) {
+    for (let i = 0; i < Math.min(tabCount, 30); i++) {
       try {
         const clicked = await page.evaluate((idx: number) => {
           const tabs = document.querySelectorAll<HTMLElement>('[role="tab"]');
@@ -5895,7 +6123,7 @@ async function interactAndCollectLinks(page: any, origin: string): Promise<strin
       document.querySelectorAll('details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]').length
     ).catch(() => 0);
 
-    for (let i = 0; i < Math.min(accordionCount, 10); i++) {
+    for (let i = 0; i < Math.min(accordionCount, 20); i++) {
       try {
         await page.evaluate((idx: number) => {
           const items = document.querySelectorAll<HTMLElement>(
@@ -5930,7 +6158,7 @@ async function expandNavMenus(page: any): Promise<boolean> {
         while (p) {
           const tag = p.tagName?.toLowerCase();
           const role = p.getAttribute('role') ?? '';
-          if (['form', 'table', 'select'].includes(tag ?? '') ||
+          if (['table', 'select'].includes(tag ?? '') ||
             ['dialog', 'alertdialog', 'combobox', 'listbox', 'grid', 'treegrid'].includes(role)) return true;
           p = p.parentElement;
         }
@@ -5963,7 +6191,7 @@ async function expandNavMenus(page: any): Promise<boolean> {
         '[role="navigation"] > ul > li, [role="menubar"] > [role="menuitem"], ' +
         '[class*="nav"] > ul > li, [class*="menu"] > li'
       ).catch(() => [] as any[]);
-      for (const h of (navHandles as any[]).slice(0, 20)) {
+      for (const h of (navHandles as any[]).slice(0, 40)) {
         try { await h.hover({ force: true, timeout: 600 }); await page.waitForTimeout(280); } catch { /* skip invisible */ }
       }
       if ((navHandles as any[]).length > 0) {
@@ -6009,6 +6237,11 @@ async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promi
         root.querySelectorAll('[data-href]').forEach((el) => {
           add(el.getAttribute('data-href') || '', el.textContent || '');
         });
+        // Form[action] GET forms: JSP/MVC apps navigate via form submissions (not anchor links)
+        root.querySelectorAll('form[action]').forEach((form) => {
+          const method = (form.getAttribute('method') || 'get').toLowerCase();
+          if (method !== 'post') add(form.getAttribute('action') || '', form.getAttribute('aria-label') || '');
+        });
         root.querySelectorAll('*').forEach((el) => {
           try {
             if (el.shadowRoot && !rootSeen.has(el.shadowRoot)) {
@@ -6470,6 +6703,30 @@ async function captureAuthDiscoveryScreen(
   if (existsSyncFs(shotPath)) {
     await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
   }
+  // GAP 11: DOM snapshot for auth-discovery phase
+  const authDomContent = await capturePageHtml(page);
+  if (authDomContent && opts.screenshotDir) {
+    const authDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
+    await fs.writeFile(authDomPath, authDomContent, 'utf8');
+    let authDomCloudUrl: string | undefined;
+    if (opts.storageConfig) {
+      try {
+        const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
+        authDomCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig as any, authDomPath, `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(authDomPath)}`);
+      } catch { /* non-fatal */ }
+    }
+    await AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, authDomPath, authDomCloudUrl).catch(() => {});
+  }
+  // Extract and save interactive elements (inputs, buttons, links) for this auth-discovery screen.
+  // Without this, login pages and auth flows always show 0 elements in Crawl Captures.
+  const authDiscoveryElements = await readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>);
+  if (authDiscoveryElements.length > 0) {
+    await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
+    for (const el of authDiscoveryElements) {
+      await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, undefined, undefined, true, undefined, undefined, undefined).catch(() => {});
+    }
+    logger.info({ screenId: screen.id, elements: authDiscoveryElements.length }, '[crawler] auth-discovery: saved elements');
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,