@detiq/crawler 1.1.18 → 1.1.20

package/dist/crawler.js

@@ -196,25 +196,53 @@ function normalizeAuthUrl(url) {
   }
 }
 async function capturePageHtml(page) {
+  let html = null;
   try {
     if (typeof page.content === "function") {
-      const html = await page.content();
-      if (typeof html === "string" && html.trim()) return html;
+      const raw = await page.content();
+      if (typeof raw === "string" && raw.trim()) html = raw;
     }
   } catch (err) {
     logger.warn({ err: String(err?.message ?? err) }, "[crawler] page.content() failed; trying DOM evaluate fallback");
   }
+  if (!html) {
+    try {
+      html = await page.evaluate(() => {
+        const type = document.doctype;
+        const doctype = type ? "<!DOCTYPE " + type.name + (type.publicId ? ' PUBLIC "' + type.publicId + '"' : "") + (type.systemId ? ' "' + type.systemId + '"' : "") + ">" : "<!DOCTYPE html>";
+        return doctype + "\n" + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? "");
+      });
+      if (typeof html !== "string" || !html.trim()) html = null;
+    } catch (err) {
+      logger.warn({ err: String(err?.message ?? err) }, "[crawler] DOM snapshot capture failed");
+      return null;
+    }
+  }
+  if (!html) return null;
   try {
-    const html = await page.evaluate(() => {
-      const type = document.doctype;
-      const doctype = type ? "<!DOCTYPE " + type.name + (type.publicId ? ' PUBLIC "' + type.publicId + '"' : "") + (type.systemId ? ' "' + type.systemId + '"' : "") + ">" : "<!DOCTYPE html>";
-      return doctype + "\n" + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? "");
+    html = await page.evaluate(() => {
+      document.querySelectorAll("script").forEach((s) => s.remove());
+      for (const sheet of Array.from(document.styleSheets)) {
+        try {
+          const cssText = Array.from(sheet.cssRules).map((r) => r.cssText).join("\n");
+          const style = document.createElement("style");
+          style.textContent = cssText;
+          const owner = sheet.ownerNode;
+          owner?.replaceWith(style);
+        } catch {
+        }
+      }
+      const doctype = (() => {
+        const dt = document.doctype;
+        return dt ? `<!DOCTYPE ${dt.name}>` : "<!DOCTYPE html>";
+      })();
+      return doctype + "\n" + (document.documentElement?.outerHTML ?? "");
     });
-    return typeof html === "string" && html.trim() ? html : null;
+    if (typeof html !== "string" || !html.trim()) html = null;
   } catch (err) {
-    logger.warn({ err: String(err?.message ?? err) }, "[crawler] DOM snapshot capture failed");
-    return null;
+    logger.warn({ err: String(err?.message ?? err) }, "[crawler] CSS inlining failed; using raw HTML");
   }
+  return html;
 }
 async function launchChromeCDP(execPath, extraArgs, proxy) {
   const port = 9200 + Math.floor(Math.random() * 700);
@@ -780,6 +808,14 @@ async function crawlProject(opts, onProgress) {
               const cloudUrl = await uploadToCloud(popupShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(popupShotPath)}`).catch(() => void 0);
               await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, popupScreen.id, popupShotPath, popupScreen.version, "DESKTOP", cloudUrl).catch(() => {
               });
+              const popupDom = await capturePageHtml(popup);
+              if (popupDom) {
+                const popupDomPath = popupShotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+                await fs.writeFile(popupDomPath, popupDom, "utf8");
+                const popupDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(popupDomPath)}`;
+                AppBrain.updateScreenDomSnapshot(opts.tenantId, popupScreen.id, popupDomPath, await uploadToCloud(popupDomPath, popupDomKey).catch(() => void 0)).catch(() => {
+                });
+              }
               const popupEls = await domFallbackElements(popup, popup).catch(() => []);
               for (const el of popupEls) {
                 AppBrain.saveElement(opts.tenantId, opts.projectId, popupScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
@@ -931,7 +967,7 @@ async function crawlProject(opts, onProgress) {
           } catch {
             return "";
           }
-        }).filter((h, i, arr) => h && arr.indexOf(h) === i).slice(0, 30);
+        }).filter((h, i, arr) => h && arr.indexOf(h) === i).slice(0, 50);
       }, _appOrigin).catch(() => []);
       for (const link of publicNavLinks) {
         if (_norm(link) === _startNorm) continue;
@@ -968,6 +1004,14 @@ async function crawlProject(opts, onProgress) {
           const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => void 0);
           await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl).catch(() => {
           });
+          const pubDom = await capturePageHtml(page);
+          if (pubDom) {
+            const pubDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+            await fs.writeFile(pubDomPath, pubDom, "utf8");
+            const pubDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(pubDomPath)}`;
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, pubDomPath, await uploadToCloud(pubDomPath, pubDomKey).catch(() => void 0)).catch(() => {
+            });
+          }
           await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
           });
           for (const el of preAuthElements) {
@@ -1369,6 +1413,8 @@ async function crawlProject(opts, onProgress) {
     let maxSeenTotal = 0;
     let lastEmittedPct = 20;
     const crawlLoopStart = Date.now();
+    const CRAWL_GLOBAL_TIMEOUT_MS = parseInt(process.env.CRAWL_GLOBAL_TIMEOUT_MS ?? "") || 2 * 60 * 60 * 1e3;
+    const isCrawlTimedOut = () => Date.now() - crawlLoopStart >= CRAWL_GLOBAL_TIMEOUT_MS;
     const startUrl = page.url();
     const resumeUrls = opts.resumeUrls ?? [];
     const gqlIntrospectionPromise = (async () => {
@@ -1398,6 +1444,10 @@ async function crawlProject(opts, onProgress) {
     ];
     const queued = new Set(seedUrls.map(canonicalize));
     const queue = new CrawlFrontier(opts.traversalStrategy ?? "BFS", seedUrls, opts.bestFirstKeywords, opts.urlFilter);
+    if (selective) {
+      queue.push = () => {
+      };
+    }
     const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? "") || 0.88);
     const deadLinkTracker = new DeadLinkTracker();
     const redirectRegistry = new RedirectRegistry();
@@ -1479,7 +1529,8 @@ async function crawlProject(opts, onProgress) {
           logger.info({ count: g11SitemapUrls.length }, "[crawler] G11: sitemap discovered public URL candidates");
           const sitemapOrigin = new URL(opts.appUrl).origin;
           let sitemapCaptured = 0;
-          for (const sUrl of g11SitemapUrls.slice(0, 30)) {
+          for (const sUrl of g11SitemapUrls) {
+            if (isCrawlTimedOut()) break;
             try {
               const u = new URL(sUrl);
               if (u.origin !== sitemapOrigin) continue;
@@ -1506,8 +1557,18 @@ async function crawlProject(opts, onProgress) {
               });
               const pubScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, pageName, sUrl, opts.jobId, "SITEMAP").catch(() => null);
               if (pubScreen) {
-                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, "DESKTOP").catch(() => {
+                const sitemapCloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => void 0);
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, "DESKTOP", sitemapCloudUrl).catch(() => {
                 });
+                const sitemapDom = await capturePageHtml(page);
+                if (sitemapDom) {
+                  const sitemapDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+                  await fs.writeFile(sitemapDomPath, sitemapDom, "utf8");
+                  const sitemapDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(sitemapDomPath)}`;
+                  const sitemapDomCloud = await uploadToCloud(sitemapDomPath, sitemapDomKey).catch(() => void 0);
+                  AppBrain.updateScreenDomSnapshot(opts.tenantId, pubScreen.id, sitemapDomPath, sitemapDomCloud).catch(() => {
+                  });
+                }
                 logger.info({ url: sUrl, name: pageName }, "[crawler] G11: captured public sitemap page");
               }
               sitemapCaptured++;
@@ -1558,6 +1619,7 @@ async function crawlProject(opts, onProgress) {
     async function runLiteWorker(workerPage) {
       while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
         if (opts.abortSignal?.aborted) break;
+        if (isCrawlTimedOut()) break;
         if (++totalAttemptsThisSegment > maxAttempts) break;
         const url = queue.shift();
         if (!url) break;
@@ -1619,6 +1681,16 @@ async function crawlProject(opts, onProgress) {
           const name = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
           visited.set(canonUrl, screen.id);
+          const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
+          const lwBodyText = await workerPage.evaluate(() => document.body?.innerText ?? "").catch(() => "");
+          if (INLINE_AUTH_WALL_RE.test(lwBodyText)) {
+            logger.warn({ url, screenId: screen.id }, "[lite-worker] Inline session-expired content detected \u2014 marking requiresAuth");
+            AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {
+            });
+            visited.set(canonUrl, "_auth_wall_");
+            if (!revisitQueue.includes(url)) revisitQueue.push(url);
+            continue;
+          }
           const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${screen.id}-desktop-v${screen.version}.png`);
           let liteElements = await domFallbackElements(workerPage).catch(() => []);
           if (liteElements.length === 0) {
@@ -1643,6 +1715,15 @@ async function crawlProject(opts, onProgress) {
             }).catch(() => {
             })
           ]);
+          const lwDom = await capturePageHtml(workerPage);
+          if (lwDom) {
+            const lwDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+            await fs.writeFile(lwDomPath, lwDom, "utf8");
+            const lwDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(lwDomPath)}`;
+            const lwDomCloud = await uploadToCloud(lwDomPath, lwDomKey).catch(() => void 0);
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, lwDomPath, lwDomCloud).catch(() => {
+            });
+          }
           const _navEls = await workerPage.evaluate(() => {
             if (typeof globalThis.__name === "undefined") globalThis.__name = (fn) => fn;
             const seen = /* @__PURE__ */ new Set();
@@ -1762,8 +1843,8 @@ async function crawlProject(opts, onProgress) {
           if (allElements.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
             });
-            for (let _ei = 0; _ei < allElements.length; _ei += 4) {
-              const _batch = allElements.slice(_ei, _ei + 4);
+            for (let _ei = 0; _ei < allElements.length; _ei += 2) {
+              const _batch = allElements.slice(_ei, _ei + 2);
               await Promise.all(_batch.map(
                 (el) => AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
                 })
@@ -1875,6 +1956,13 @@ async function crawlProject(opts, onProgress) {
         logger.info("Abort signal received \u2014 stopping crawl");
         throw new Error("CRAWL_ABORTED");
       }
+      if (isCrawlTimedOut()) {
+        const elapsedMin = Math.round((Date.now() - crawlLoopStart) / 6e4);
+        logger.warn({ elapsedMin, visited: visited.size, queued: queue.length }, "[crawler] Global timeout reached \u2014 stopping BFS to prevent runaway crawl");
+        await onProgress?.(89, `Crawl timeout after ${elapsedMin}min \u2014 ${visited.size} pages captured`).catch(() => {
+        });
+        break;
+      }
       if (opts.autoThrottle && navCount > 0) {
         const delayMs = Math.min(5e3, Math.max(0, avgNavMs * 0.5));
         if (delayMs > 100) await page.waitForTimeout(delayMs).catch(() => {
@@ -2069,8 +2157,18 @@ async function crawlProject(opts, onProgress) {
               }).catch(() => "Modal");
               const modalScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, modalName, url, opts.jobId, discoveryMethodFor(url)).catch(() => null);
               if (modalScreen) {
-                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, "DESKTOP").catch(() => {
+                const modalCloudUrl = await uploadToCloud(modalShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(modalShotPath)}`).catch(() => void 0);
+                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, "DESKTOP", modalCloudUrl).catch(() => {
                 });
+                const modalDom = await capturePageHtml(page);
+                if (modalDom) {
+                  const modalDomPath = modalShotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+                  await fs.writeFile(modalDomPath, modalDom, "utf8");
+                  const modalDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(modalDomPath)}`;
+                  const modalDomCloud = await uploadToCloud(modalDomPath, modalDomKey).catch(() => void 0);
+                  AppBrain.updateScreenDomSnapshot(opts.tenantId, modalScreen.id, modalDomPath, modalDomCloud).catch(() => {
+                  });
+                }
                 const modalElements = await domFallbackElements(page, rawPage ?? void 0).catch(() => []);
                 for (const el of modalElements) {
                   AppBrain.saveElement(opts.tenantId, opts.projectId, modalScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
@@ -2115,6 +2213,11 @@ async function crawlProject(opts, onProgress) {
                 }
               }).catch(() => {
               });
+              await opts.onCheckpoint?.(
+                Array.from(visited.entries()).filter(([_k, v]) => v !== "_pending_").map(([k]) => k),
+                queue.filter((u) => !visited.has(canonicalize(u)))
+              ).catch(() => {
+              });
               throw new Error(`[AUTH_FAILURE] Login wall detected: ${loginWallRedirectCount} URLs redirected to the login page. Add credentials in project settings and retry the crawl.`);
             }
           }
@@ -2150,6 +2253,11 @@ async function crawlProject(opts, onProgress) {
                 payload: { url, landed: landedUrl, reason: "Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl." }
               }).catch(() => {
               });
+              await opts.onCheckpoint?.(
+                Array.from(visited.entries()).filter(([_k, v]) => v !== "_pending_").map(([k]) => k),
+                queue.filter((u) => !visited.has(canonicalize(u)))
+              ).catch(() => {
+              });
               throw new Error("[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.");
             }
           } else {
@@ -2164,6 +2272,11 @@ async function crawlProject(opts, onProgress) {
               payload: { url, landed: landedUrl, reason: "Session expired and no credentials are configured. Add credentials in project settings and re-capture the session." }
             }).catch(() => {
             });
+            await opts.onCheckpoint?.(
+              Array.from(visited.entries()).filter(([_k, v]) => v !== "_pending_").map(([k]) => k),
+              queue.filter((u) => !visited.has(canonicalize(u)))
+            ).catch(() => {
+            });
             throw new Error("[AUTH_FAILURE] Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.");
           }
         }
@@ -2762,120 +2875,139 @@ async function crawlProject(opts, onProgress) {
             })();
           }
         }
-        try {
-          const domContent = await capturePageHtml(page);
-          if (!domContent) {
-            logger.warn({ url, screenId: screen.id }, "[crawler] markdown extraction skipped: DOM snapshot unavailable");
-          } else {
-            const domPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
-            await fs.writeFile(domPath, domContent, "utf8");
-            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath).catch(() => {
-            });
-            const [iframeContent, shadowContent] = await Promise.all([
-              extractIframeContent(page).catch(() => []),
-              pierceShadowDom(page).catch(() => "")
-            ]);
-            const supplemental = [
-              ...iframeContent.map((f) => f.markdown).filter(Boolean),
-              shadowContent
-            ].join("\n\n").trim();
-            const extracted = extractMarkdown(
-              supplemental ? domContent + "\n<!-- iframe/shadow -->\n" + supplemental.slice(0, 3e3) : domContent,
-              url
-            );
-            let fitMarkdown;
-            if (extracted) {
-              const mdPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".md");
-              fitMarkdown = buildFitMarkdown(extracted.markdown, [
-                ...domainProfile.importantScreenPatterns,
-                ...domainProfile.workflowGoals,
-                ...opts.bestFirstKeywords ?? []
+        const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
+        const pageBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
+        if (INLINE_AUTH_WALL_RE.test(pageBodyText)) {
+          logger.warn({ url, screenId: screen.id }, "[crawler] Inline session-expired content detected \u2014 skipping DOM snapshot, marking requiresAuth");
+          AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {
+          });
+          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+            screenId: screen.id,
+            jobId: opts.jobId,
+            type: "AUTH_FAILURE",
+            payload: { url, reason: "Inline session-expired page rendered without URL redirect. Re-authenticate and re-crawl." }
+          }).catch(() => {
+          });
+        } else {
+          try {
+            const domContent = await capturePageHtml(page);
+            if (!domContent) {
+              logger.warn({ url, screenId: screen.id }, "[crawler] markdown extraction skipped: DOM snapshot unavailable");
+            } else {
+              const domPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+              await fs.writeFile(domPath, domContent, "utf8");
+              (async () => {
+                const domKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(domPath)}`;
+                const domCloudUrl = await uploadToCloud(domPath, domKey).catch(() => void 0);
+                AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath, domCloudUrl).catch(() => {
+                });
+              })();
+              const [iframeContent, shadowContent] = await Promise.all([
+                extractIframeContent(page).catch(() => []),
+                pierceShadowDom(page).catch(() => "")
               ]);
-              const { text: scrubbed, scrubbed: hadPii } = scrubPii(extracted.markdown);
-              if (hadPii) logger.info({ url }, "[crawler] PII detected and redacted from markdown");
-              const dedupResult = deduplicator.check(url, scrubbed);
-              if (dedupResult.isDuplicate) {
-                logger.info({ url, similarUrl: dedupResult.similarUrl, score: dedupResult.similarityScore }, "[crawler] near-duplicate page skipped");
-              }
-              await fs.writeFile(mdPath, scrubbed, "utf8");
-              const summaryPromise = (async () => {
-                try {
-                  const llmCfg = await getCachedCrawlLLM(opts.tenantId);
-                  const snippet = extracted.markdown.slice(0, 1200);
-                  if (snippet.trim().length < 80) return void 0;
-                  const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ""}:page-summary:${url}`);
-                  const { text, usage } = await askTracked(
-                    "You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.",
-                    `Screen: "${name}"
+              const supplemental = [
+                ...iframeContent.map((f) => f.markdown).filter(Boolean),
+                shadowContent
+              ].join("\n\n").trim();
+              const extracted = extractMarkdown(
+                supplemental ? domContent + "\n<!-- iframe/shadow -->\n" + supplemental.slice(0, 3e3) : domContent,
+                url
+              );
+              let fitMarkdown;
+              if (extracted) {
+                const mdPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".md");
+                fitMarkdown = buildFitMarkdown(extracted.markdown, [
+                  ...domainProfile.importantScreenPatterns,
+                  ...domainProfile.workflowGoals,
+                  ...opts.bestFirstKeywords ?? []
+                ]);
+                const { text: scrubbed, scrubbed: hadPii } = scrubPii(extracted.markdown);
+                if (hadPii) logger.info({ url }, "[crawler] PII detected and redacted from markdown");
+                const dedupResult = deduplicator.check(url, scrubbed);
+                if (dedupResult.isDuplicate) {
+                  logger.info({ url, similarUrl: dedupResult.similarUrl, score: dedupResult.similarityScore }, "[crawler] near-duplicate page skipped");
+                }
+                await fs.writeFile(mdPath, scrubbed, "utf8");
+                const summaryPromise = (async () => {
+                  try {
+                    const llmCfg = await getCachedCrawlLLM(opts.tenantId);
+                    const snippet = extracted.markdown.slice(0, 1200);
+                    if (snippet.trim().length < 80) return void 0;
+                    const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ""}:page-summary:${url}`);
+                    const { text, usage } = await askTracked(
+                      "You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.",
+                      `Screen: "${name}"
 URL: ${url}
 
 Content:
 ${snippet}`,
-                    llmCfg,
-                    200,
-                    void 0,
-                    { temperature: 0, seed: summarySeed }
-                  );
-                  if (usage) {
-                    recordLLMCall({
-                      tenantId: opts.tenantId,
-                      projectId: opts.projectId,
-                      jobId: opts.jobId,
-                      agentType: "crawl_summary",
-                      label: `page_summary:${name}`,
-                      usage
-                    }).catch(() => {
-                    });
+                      llmCfg,
+                      200,
+                      void 0,
+                      { temperature: 0, seed: summarySeed }
+                    );
+                    if (usage) {
+                      recordLLMCall({
+                        tenantId: opts.tenantId,
+                        projectId: opts.projectId,
+                        jobId: opts.jobId,
+                        agentType: "crawl_summary",
+                        label: `page_summary:${name}`,
+                        usage
+                      }).catch(() => {
+                      });
+                    }
+                    const clean = text.trim().replace(/^["']|["']$/g, "");
+                    const firstSentence = clean.match(/^[^.!?]+[.!?]/)?.[0] ?? clean;
+                    return firstSentence.trim();
+                  } catch {
+                    return void 0;
                   }
-                  const clean = text.trim().replace(/^["']|["']$/g, "");
-                  const firstSentence = clean.match(/^[^.!?]+[.!?]/)?.[0] ?? clean;
-                  return firstSentence.trim();
-                } catch {
-                  return void 0;
-                }
-              })();
-              AppBrain.updateScreenMarkdown(opts.tenantId, opts.projectId, screen.id, {
-                markdownPath: mdPath,
-                markdownMethod: extracted.method,
-                markdownWordCount: extracted.wordCount,
-                markdown: fitMarkdown ?? extracted.markdown,
-                heading: extracted.title || name
-              }).catch((err) => console.warn("[crawler] markdown extraction save failed:", err));
-              summaryPromise.then((aiSummary) => {
-                if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => {
+                })();
+                AppBrain.updateScreenMarkdown(opts.tenantId, opts.projectId, screen.id, {
+                  markdownPath: mdPath,
+                  markdownMethod: extracted.method,
+                  markdownWordCount: extracted.wordCount,
+                  markdown: fitMarkdown ?? extracted.markdown,
+                  heading: extracted.title || name
+                }).catch((err) => console.warn("[crawler] markdown extraction save failed:", err));
+                summaryPromise.then((aiSummary) => {
+                  if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => {
+                  });
                 });
+              }
+              const structuredJson = summarizeStructuredElements(atElements);
+              const artifact = buildCrawlArtifactEnvelope({
+                tenantId: opts.tenantId,
+                projectId: opts.projectId,
+                jobId: opts.jobId,
+                url,
+                canonicalUrl: canonicalize(url),
+                screenId: screen.id,
+                screenshotPath: shotPath,
+                domPath,
+                domContent,
+                markdown: extracted?.markdown,
+                fitMarkdown,
+                accessibilityElements: atElements,
+                structuredJson,
+                evidence: [
+                  { type: "domain_profile", payload: { industry: domainProfile.industry, confidence: domainProfile.confidence } },
+                  { type: "engine_route", payload: routeDecision },
+                  { type: "screen_fingerprints", payload: { urlCanonicalHash: sha256(canonicalize(url)) } }
+                ]
               });
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                screenId: screen.id,
+                jobId: opts.jobId,
+                type: "SCREEN_ARTIFACT_ENVELOPE",
+                payload: artifact
+              }).catch((err) => console.warn("[crawler] artifact evidence save failed:", err?.message ?? err));
             }
-            const structuredJson = summarizeStructuredElements(atElements);
-            const artifact = buildCrawlArtifactEnvelope({
-              tenantId: opts.tenantId,
-              projectId: opts.projectId,
-              jobId: opts.jobId,
-              url,
-              canonicalUrl: canonicalize(url),
-              screenId: screen.id,
-              screenshotPath: shotPath,
-              domPath,
-              domContent,
-              markdown: extracted?.markdown,
-              fitMarkdown,
-              accessibilityElements: atElements,
-              structuredJson,
-              evidence: [
-                { type: "domain_profile", payload: { industry: domainProfile.industry, confidence: domainProfile.confidence } },
-                { type: "engine_route", payload: routeDecision },
-                { type: "screen_fingerprints", payload: { urlCanonicalHash: sha256(canonicalize(url)) } }
-              ]
-            });
-            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
-              screenId: screen.id,
-              jobId: opts.jobId,
-              type: "SCREEN_ARTIFACT_ENVELOPE",
-              payload: artifact
-            }).catch((err) => console.warn("[crawler] artifact evidence save failed:", err?.message ?? err));
+          } catch (err) {
+            logger.warn({ url, screenId: screen.id, err: String(err?.message ?? err) }, "[crawler] markdown extraction failed");
           }
-        } catch (err) {
-          logger.warn({ url, screenId: screen.id, err: String(err?.message ?? err) }, "[crawler] markdown extraction failed");
         }
         if (a11yScore !== null) {
           AppBrain.updateScreenA11y(opts.tenantId, screen.id, { a11yScore }).catch((err) => console.warn("[crawler] a11y score update failed:", err));
@@ -3046,6 +3178,8 @@ ${snippet}`,
         const driftedToAuthWall = !weWantedThisPage && (BFS_LOGIN_PATH.test(postExtractParsed.pathname) || BFS_ERROR_PATH.test(postExtractParsed.pathname));
         if (driftedToAuthWall) {
           logger.warn({ url, postExtractUrl }, "[crawler] Page drifted to auth wall during element extraction \u2014 skipping element save to prevent login-page element contamination");
+          AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {
+          });
           AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
             screenId: screen.id,
             jobId: opts.jobId,
@@ -3063,8 +3197,8 @@ ${snippet}`,
           } else {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
             });
-            for (let _ei = 0; _ei < elements.length; _ei += 4) {
-              const _batch = elements.slice(_ei, _ei + 4);
+            for (let _ei = 0; _ei < elements.length; _ei += 2) {
+              const _batch = elements.slice(_ei, _ei + 2);
               await Promise.all(_batch.map(async (el) => {
                 const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark);
                 const label = el.meaning;
@@ -3075,7 +3209,7 @@ ${snippet}`,
                   { type: "aria-label", value: label, confidence: 0.85 },
                   { type: "tag-hint", value: selectorHint, confidence: 0.6 }
                 ];
-                AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+                await AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
               }));
             }
           }
@@ -3104,7 +3238,7 @@ ${snippet}`,
               try {
                 const targetPath = new URL(lm.href).pathname;
                 const canonLmHref = canonicalize(lm.href);
-                if (!SKIP_PATH.test(targetPath) && isUsefulNavigationUrl(lm.href, origin) && !queued.has(canonLmHref) && !visited.has(canonLmHref) && queued.size < maxScreens * 2) {
+                if (!SKIP_PATH.test(targetPath) && isUsefulNavigationUrl(lm.href, origin) && !queued.has(canonLmHref) && !visited.has(canonLmHref) && queued.size < maxScreens * 3) {
                   if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
                   queued.add(canonLmHref);
                   queue.push(lm.href, { parentElements: elements.length });
@@ -3125,7 +3259,7 @@ ${snippet}`,
               } catch {
               }
               const canonHref = canonicalize(href);
-              if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 2) {
+              if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 3) {
                 try {
                   const hrefPath = new URL(href).pathname;
                   if (isGeneratedDynamicPath(hrefPath)) continue;
@@ -3178,7 +3312,7 @@ ${snippet}`,
                 const jsDetectedRoutes = await detectJsRoutes(page, origin);
                 for (const route of jsDetectedRoutes) {
                   const canonRoute = canonicalize(route);
-                  if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 2) {
+                  if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 3) {
                     queued.add(canonRoute);
                     queue.push(route, { parentElements: elements.length });
                     logger.info({ route }, "[P2] JS router: discovered route added to queue");
@@ -3192,7 +3326,7 @@ ${snippet}`,
               try {
                 const abs = `${origin}${relPath}`;
                 const canon = canonicalize(abs);
-                if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
                   queued.add(canon);
                   queue.push(abs);
                   logger.info({ url: abs }, "[P4] XHR: page URL from JSON response added to queue");
@@ -3200,7 +3334,7 @@ ${snippet}`,
               } catch {
               }
             }
-            if (nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? "") || 5) && opts.skyvernConfig) {
+            if (!selective && nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? "") || 5) && opts.skyvernConfig) {
               try {
                 logger.info({ url }, "Gap 5: thin page detected \u2014 asking Skyvern to discover navigation URLs");
                 const skyvernNav = await skyvernNavigate(
@@ -3210,7 +3344,7 @@ ${snippet}`,
                 );
                 for (const skyvernUrl of skyvernNav.visitedUrls) {
                   const canonSkyvernUrl = canonicalize(skyvernUrl);
-                  if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 2) {
+                  if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 3) {
                     logger.info({ url: skyvernUrl }, "Gap 5: Skyvern-discovered URL added to ZeTa crawl queue");
                     queued.add(canonSkyvernUrl);
                     queue.push(skyvernUrl);
@@ -3265,9 +3399,9 @@ ${snippet}`,
         lastEmittedPct = clampedPct;
         await onProgress?.(clampedPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
         logger.info({ name, elements: elements.length }, "Screen crawled");
-        if (++checkpointSuccessCount % 10 === 0) {
+        if (++checkpointSuccessCount % 5 === 0) {
           await opts.onCheckpoint?.(
-            Array.from(visited.keys()),
+            Array.from(visited.entries()).filter(([_k, v]) => v !== "_pending_").map(([k]) => k),
             queue.filter((u) => !visited.has(canonicalize(u)))
           );
         }
@@ -3295,7 +3429,7 @@ ${snippet}`,
     }
     const successfulScreenCount = discovered.filter((d) => d.screenId).length;
     const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? "") || 10;
-    if (successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
+    if (!selective && successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
       await onProgress?.(78, "Few screens found \u2014 using Skyvern to discover additional navigation URLs");
       const currentUrl = page.url();
       const skyvernNav = await skyvernNavigate(
@@ -3310,7 +3444,7 @@ ${snippet}`,
       if (skyvernUrls.length > 0) {
         logger.info({ count: skyvernUrls.length }, "Gap 5: Skyvern found new URLs \u2014 processing with ZeTa native crawler");
         for (const skyvernUrl of skyvernUrls) {
-          if (visited.size >= maxScreens) break;
+          if (isCrawlTimedOut() || visited.size >= maxScreens) break;
           try {
             await page.goto(skyvernUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
             await new Promise((r) => setTimeout(r, 1500));
@@ -3319,7 +3453,17 @@ ${snippet}`,
             const shotPath = `${opts.screenshotDir ?? "/tmp"}/${screen.id}-v${screen.version}.png`;
             await waitForScreenshotReady(page, rawPage ?? void 0);
             await page.screenshot({ path: shotPath, fullPage: true });
-            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP");
+            const skyvernCloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => void 0);
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", skyvernCloudUrl);
+            const skyvernDom = await capturePageHtml(page);
+            if (skyvernDom) {
+              const skyvernDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+              await fs.writeFile(skyvernDomPath, skyvernDom, "utf8");
+              const skyvernDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(skyvernDomPath)}`;
+              const skyvernDomCloud = await uploadToCloud(skyvernDomPath, skyvernDomKey).catch(() => void 0);
+              AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, skyvernDomPath, skyvernDomCloud).catch(() => {
+              });
+            }
             const [elems, links, linkMappings] = await Promise.all([
               extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0),
               discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0),
@@ -3345,7 +3489,7 @@ ${snippet}`,
             logger.warn({ url: skyvernUrl, err: String(err) }, "Gap 5: ZeTa crawl of Skyvern URL failed");
           }
         }
-        while (queue.length > 0 && visited.size < maxScreens) {
+        while (queue.length > 0 && visited.size < maxScreens && !isCrawlTimedOut()) {
           const href = queue.shift();
           const canonHref = canonicalize(href);
           if (visited.has(canonHref)) continue;
@@ -3358,7 +3502,17 @@ ${snippet}`,
             const sp = `${opts.screenshotDir ?? "/tmp"}/${sc.id}-v${sc.version}.png`;
             await waitForScreenshotReady(page, rawPage ?? void 0);
             await page.screenshot({ path: sp, fullPage: true });
-            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, "DESKTOP");
+            const skyvernDrainCloudUrl = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => void 0);
+            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, "DESKTOP", skyvernDrainCloudUrl);
+            const skyvernDrainDom = await capturePageHtml(page);
+            if (skyvernDrainDom) {
+              const skyvernDrainDomPath = sp.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+              await fs.writeFile(skyvernDrainDomPath, skyvernDrainDom, "utf8");
+              const skyvernDrainDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(skyvernDrainDomPath)}`;
+              const skyvernDrainDomCloud = await uploadToCloud(skyvernDrainDomPath, skyvernDrainDomKey).catch(() => void 0);
+              AppBrain.updateScreenDomSnapshot(opts.tenantId, sc.id, skyvernDrainDomPath, skyvernDrainDomCloud).catch(() => {
+              });
+            }
             const [elems2, linkMappings2] = await Promise.all([
               extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0),
               extractLinkMappings(page, origin)
@@ -3413,7 +3567,8 @@ ${snippet}`,
     if (revisitQueue.length > 0) {
       await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content\u2026`);
       logger.info({ count: revisitQueue.length }, "[P3a] Starting async content re-visit pass");
-      for (const revisitUrl of revisitQueue.slice(0, 10)) {
+      for (const revisitUrl of revisitQueue.slice(0, 20)) {
+        if (isCrawlTimedOut()) break;
         const rawScreenId = visited.get(canonicalize(revisitUrl));
         const isAuthWallRevisit = rawScreenId === "_auth_wall_";
         if (!rawScreenId) continue;
@@ -3439,8 +3594,18 @@ ${snippet}`,
           }
           const shotPath2 = `${opts.screenshotDir ?? "/tmp"}/${existingScreenId}-revisit.png`;
           await page.screenshot({ path: shotPath2, fullPage: true });
-          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, "DESKTOP", void 0).catch(() => {
+          const p3aCloudUrl = await uploadToCloud(shotPath2, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath2)}`).catch(() => void 0);
+          await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, "DESKTOP", p3aCloudUrl).catch(() => {
           });
+          const p3aDom = await capturePageHtml(page);
+          if (p3aDom) {
+            const p3aDomPath = shotPath2.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+            await fs.writeFile(p3aDomPath, p3aDom, "utf8");
+            const p3aDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(p3aDomPath)}`;
+            const p3aDomCloud = await uploadToCloud(p3aDomPath, p3aDomKey).catch(() => void 0);
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, existingScreenId, p3aDomPath, p3aDomCloud).catch(() => {
+            });
+          }
           const revisitElements = await extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []);
           if (revisitElements.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, existingScreenId).catch(() => {
@@ -3453,7 +3618,7 @@ ${snippet}`,
           const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []);
           for (const href of revisitLinks) {
             const canon = canonicalize(href);
-            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
               queued.add(canon);
               queue.push(href);
             }
@@ -3469,6 +3634,7 @@ ${snippet}`,
       logger.info({ newLinks: p3bCount }, "[P3b] Extended BFS \u2014 processing links discovered during revisit pass");
       await onProgress?.(82, `Extended crawl: ${p3bCount} newly discovered page(s)\u2026`);
       while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        if (isCrawlTimedOut()) break;
         const href = queue.shift();
         if (!href) break;
         const canonHref = canonicalize(href);
@@ -3489,6 +3655,15 @@ ${snippet}`,
           const cloudUrl3 = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => "");
           await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, sp, screen.version, "DESKTOP", cloudUrl3).catch(() => {
           });
+          const p3bDom = await capturePageHtml(page);
+          if (p3bDom) {
+            const p3bDomPath = sp.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+            await fs.writeFile(p3bDomPath, p3bDom, "utf8");
+            const p3bDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(p3bDomPath)}`;
+            const p3bDomCloud = await uploadToCloud(p3bDomPath, p3bDomKey).catch(() => void 0);
+            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, p3bDomPath, p3bDomCloud).catch(() => {
+            });
+          }
           const [p3bElems, p3bLinks] = await Promise.all([
             extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []),
             discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).catch(() => [])
@@ -3496,8 +3671,8 @@ ${snippet}`,
           if (p3bElems.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
             });
-            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 4) {
-              const _b3 = p3bElems.slice(_ei3, _ei3 + 4);
+            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 2) {
+              const _b3 = p3bElems.slice(_ei3, _ei3 + 2);
               await Promise.all(_b3.map(
                 (el) => AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
                 })
@@ -3506,7 +3681,7 @@ ${snippet}`,
           }
           for (const link of p3bLinks) {
             const c = canonicalize(link);
-            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 2) {
+            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 3) {
               queued.add(c);
               queue.push(link);
             }
@@ -3543,9 +3718,9 @@ ${snippet}`,
           });
           logger.info({ confirmed: confirmedGaps.length, total: gapCandidates.length }, "[P6] Gap pass: visiting confirmed gap URLs");
           const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
-          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? "") || 8)) {
+          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? "") || 25)) {
+            if (isCrawlTimedOut() || visited.size >= maxScreens) break;
             const gapUrl = gap.confirmedUrl;
-            if (visited.size >= maxScreens) break;
             try {
               await page.goto(gapUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
               if (gapLoginPattern.test(page.url())) {
@@ -3558,7 +3733,17 @@ ${snippet}`,
               const gapShot = `${opts.screenshotDir ?? "/tmp"}/${gapScreen.id}-v${gapScreen.version}.png`;
               await waitForScreenshotReady(page, rawPage ?? void 0);
               await page.screenshot({ path: gapShot, fullPage: true });
-              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, "DESKTOP");
+              const gapCloudUrl = await uploadToCloud(gapShot, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(gapShot)}`).catch(() => void 0);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, "DESKTOP", gapCloudUrl);
+              const gapDom = await capturePageHtml(page);
+              if (gapDom) {
+                const gapDomPath = gapShot.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+                await fs.writeFile(gapDomPath, gapDom, "utf8");
+                const gapDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(gapDomPath)}`;
+                const gapDomCloud = await uploadToCloud(gapDomPath, gapDomKey).catch(() => void 0);
+                AppBrain.updateScreenDomSnapshot(opts.tenantId, gapScreen.id, gapDomPath, gapDomCloud).catch(() => {
+                });
+              }
               const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
                 extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []),
                 discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []),
@@ -3575,7 +3760,7 @@ ${snippet}`,
               }
               for (const href of gapLinks) {
                 const canon = canonicalize(href);
-                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 2) {
+                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
                   queued.add(canon);
                   queue.push(href);
                 }
@@ -3777,6 +3962,7 @@ ${snippet}`,
           });
           const mobilePage = await mobileCtx.newPage();
           for (const disc of discovered) {
+            if (isCrawlTimedOut()) break;
             if (!disc.screenId) continue;
             try {
               await mobilePage.goto(disc.url, { waitUntil: "domcontentloaded", timeout: 2e4 });
@@ -3938,8 +4124,14 @@ ${snippet}`,
       }
     })();
     await onProgress?.(98, "Finalizing");
+    const wasTimeout = isCrawlTimedOut();
     const pendingUrls = queue.filter((u) => !visited.has(canonicalize(u)));
+    const visitedUrls = Array.from(visited.entries()).filter(([_k, v]) => v !== "_pending_").map(([k]) => k);
     const hasMoreSegments = pendingUrls.length > 0;
+    if (wasTimeout && pendingUrls.length > 0) {
+      const elapsedMin = Math.round((Date.now() - crawlLoopStart) / 6e4);
+      crawlWarnings.push(`global-timeout: crawl stopped after ${elapsedMin} minutes with ${pendingUrls.length} URLs remaining`);
+    }
     const screensSucceeded = discovered.filter((d) => !!d.screenId).length;
     const screensErrored = discovered.filter((d) => !!d.error && !d.screenId).length;
     if (screensSucceeded >= 3) {
@@ -3963,6 +4155,10 @@ ${snippet}`,
       performanceMetrics,
       pendingUrls,
       // feed into next segment's resumeUrls
+      visitedUrls,
+      // all fully-processed URLs for skipUrls in continuation
+      wasTimeout,
+      // true when global timeout triggered the BFS break
       segmentIndex: segIdx,
       // which segment this was
       hasMoreSegments,
@@ -4016,7 +4212,7 @@ async function extractElements(stagehand, page, llmCfg, rawPage) {
       const iframeUrl = await frame.url().catch(() => "");
       if (!iframeUrl || iframeUrl === "about:blank" || iframeUrl.startsWith("javascript:")) continue;
       const frameEls = await domFallbackElements(frame).catch(() => []);
-      for (const el of frameEls.slice(0, 10)) {
+      for (const el of frameEls.slice(0, 30)) {
         iframeElements.push({ ...el, notes: el.notes ? `${el.notes} [in iframe]` : "[in iframe]" });
       }
     }
@@ -4036,7 +4232,7 @@ async function extractElements(stagehand, page, llmCfg, rawPage) {
           seen.add(label);
           out.push({ meaning: label, role, notes: "[dynamic/revealed]" });
         });
-        return out.slice(0, 10);
+        return out.slice(0, 30);
       }).catch(() => []);
     } catch {
       dynamicElements = [];
@@ -4081,7 +4277,7 @@ async function extractElements(stagehand, page, llmCfg, rawPage) {
     ]);
     if (!extracted) return domElements;
     const filtered = extracted.elements.filter((e) => !ELEMENT_NOISE.test(e.meaning ?? ""));
-    return filtered.slice(0, 20);
+    return filtered.slice(0, 40);
   } catch {
     return domElements;
   }
@@ -4104,7 +4300,7 @@ async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false, ra
     });
   }
   const allDomLinks = Array.from(/* @__PURE__ */ new Set([...domLinks, ...interactiveLinks]));
-  if (allDomLinks.length >= 30 || skipAi) return allDomLinks;
+  if (allDomLinks.length >= 50 || skipAi) return allDomLinks;
   try {
     const extracted = await Promise.race([
       stagehand.extract(
@@ -4426,7 +4622,7 @@ async function contentFallbackElements(page) {
     const mainContent = document.querySelector('main, [role="main"], article') ?? document.body;
     let listItemCount = 0;
     mainContent.querySelectorAll("li").forEach((li) => {
-      if (listItemCount >= 30) return;
+      if (listItemCount >= 50) return;
       if (!isVisible(li)) return;
       const text = li.textContent?.replace(/\s+/g, " ").trim().slice(0, 150);
       if (text && text.length > 3 && text.length < 200) {
@@ -4641,7 +4837,7 @@ async function contentFallbackElements(page) {
     const navTags = /* @__PURE__ */ new Set(["nav", "header", "footer"]);
     let contentLinkCount = 0;
     document.querySelectorAll("a[href]").forEach((a) => {
-      if (contentLinkCount >= 20) return;
+      if (contentLinkCount >= 40) return;
       if (!isVisible(a)) return;
       let p = a.parentElement;
       while (p) {
@@ -5084,7 +5280,7 @@ async function interactAndCollectLinks(page, origin) {
     const tabCount = await page.evaluate(
       () => document.querySelectorAll('[role="tab"]').length
     ).catch(() => 0);
-    for (let i = 0; i < Math.min(tabCount, 15); i++) {
+    for (let i = 0; i < Math.min(tabCount, 30); i++) {
       try {
         const clicked = await page.evaluate((idx) => {
           const tabs = document.querySelectorAll('[role="tab"]');
@@ -5128,7 +5324,7 @@ async function interactAndCollectLinks(page, origin) {
     const accordionCount = await page.evaluate(
       () => document.querySelectorAll('details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]').length
     ).catch(() => 0);
-    for (let i = 0; i < Math.min(accordionCount, 10); i++) {
+    for (let i = 0; i < Math.min(accordionCount, 20); i++) {
       try {
         await page.evaluate((idx) => {
           const items = document.querySelectorAll(
@@ -5166,7 +5362,7 @@ async function expandNavMenus(page) {
         while (p) {
           const tag = p.tagName?.toLowerCase();
           const role = p.getAttribute("role") ?? "";
-          if (["form", "table", "select"].includes(tag ?? "") || ["dialog", "alertdialog", "combobox", "listbox", "grid", "treegrid"].includes(role)) return true;
+          if (["table", "select"].includes(tag ?? "") || ["dialog", "alertdialog", "combobox", "listbox", "grid", "treegrid"].includes(role)) return true;
           p = p.parentElement;
         }
         return false;
@@ -5196,7 +5392,7 @@ async function expandNavMenus(page) {
       const navHandles = await page.$$(
         'nav > ul > li, nav > li, header > ul > li, header > nav > ul > li, [role="navigation"] > ul > li, [role="menubar"] > [role="menuitem"], [class*="nav"] > ul > li, [class*="menu"] > li'
       ).catch(() => []);
-      for (const h of navHandles.slice(0, 20)) {
+      for (const h of navHandles.slice(0, 40)) {
         try {
           await h.hover({ force: true, timeout: 600 });
           await page.waitForTimeout(280);
@@ -5245,6 +5441,10 @@ async function domFallbackLinks(page, origin, rawPage) {
         root.querySelectorAll("[data-href]").forEach((el) => {
           add(el.getAttribute("data-href") || "", el.textContent || "");
         });
+        root.querySelectorAll("form[action]").forEach((form) => {
+          const method = (form.getAttribute("method") || "get").toLowerCase();
+          if (method !== "post") add(form.getAttribute("action") || "", form.getAttribute("aria-label") || "");
+        });
         root.querySelectorAll("*").forEach((el) => {
           try {
             if (el.shadowRoot && !rootSeen.has(el.shadowRoot)) {
@@ -5652,6 +5852,21 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
     await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", authCloudUrl).catch(() => {
     });
   }
+  const authDomContent = await capturePageHtml(page);
+  if (authDomContent && opts.screenshotDir) {
+    const authDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, ".dom.html");
+    await fs.writeFile(authDomPath, authDomContent, "utf8");
+    let authDomCloudUrl;
+    if (opts.storageConfig) {
+      try {
+        const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
+        authDomCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig, authDomPath, `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(authDomPath)}`);
+      } catch {
+      }
+    }
+    await AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, authDomPath, authDomCloudUrl).catch(() => {
+    });
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,