@detiq/crawler 0.1.5 → 0.1.6

package/dist/crawler.js

@@ -1002,7 +1002,8 @@ async function crawlProject(opts, onProgress) {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0;
-    let lastEmittedPct = 20;
+    let lastEmittedPct = 5;
+    const crawlLoopStart = Date.now();
     const startUrl = page.url();
     const resumeUrls = opts.resumeUrls ?? [];
     const gqlIntrospectionPromise = (async () => {
@@ -1708,7 +1709,11 @@ async function crawlProject(opts, onProgress) {
         if (cachedLinks) {
           logger.info({ url, cachedCount: cachedLinks.length }, "[crawler] link-discovery: using cached recipe \u2014 AI will still run and merge");
         }
-        await onProgress?.(20, `Crawling ${visited.size} of ~${visited.size + queue.length} \xB7 ${(() => {
+        const elapsedDiscoverySec = (Date.now() - crawlLoopStart) / 1e3;
+        const discoveryPct = Math.round(5 + Math.min(15, elapsedDiscoverySec / 60 * 15));
+        const preCapturePct = Math.max(lastEmittedPct, discoveryPct);
+        lastEmittedPct = preCapturePct;
+        await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} \xB7 ${(() => {
           try {
             return new URL(url).pathname || "/";
           } catch {
@@ -1719,6 +1724,13 @@ async function crawlProject(opts, onProgress) {
           () => document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
         ).catch(() => 0);
         await waitForScreenshotReady(page);
+        await onProgress?.(lastEmittedPct, `Capturing ${(() => {
+          try {
+            return new URL(url).pathname || "/";
+          } catch {
+            return "/";
+          }
+        })()}`);
         const [, elements, nextLinks, atElements] = await Promise.all([
           page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")),
           extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []),
@@ -2158,7 +2170,8 @@ ${snippet}`,
           })();
         }
         discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
-        maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
+        const effectiveMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+        maxSeenTotal = Math.max(maxSeenTotal, effectiveMax);
         const rawPct = 20 + Math.round(Math.min(60, visited.size / Math.max(1, maxSeenTotal) * 60));
         const clampedPct = Math.max(lastEmittedPct, rawPct);
         lastEmittedPct = clampedPct;
@@ -3399,12 +3412,14 @@ async function waitForStableDOM(page, maxWait = 8e3, stabilityMs = 400) {
   }
 }
 async function waitForScreenshotReady(page) {
+  const hardDeadline = Date.now() + 2e4;
+  const withHardCap = (p) => Promise.race([p, new Promise((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
   try {
-    await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => {
-    });
-    await page.waitForLoadState("networkidle", { timeout: 12e3 }).catch(() => {
-    });
-    await page.evaluate(() => {
+    await withHardCap(page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => {
+    }));
+    await withHardCap(page.waitForLoadState("networkidle", { timeout: 12e3 }).catch(() => {
+    }));
+    await withHardCap(page.evaluate(() => {
       if (document.readyState === "complete") return true;
       return new Promise((resolve) => {
         const done = () => resolve(true);
@@ -3412,15 +3427,15 @@ async function waitForScreenshotReady(page) {
         setTimeout(done, 4e3);
       });
     }).catch(() => {
-    });
-    await page.evaluate(async () => {
+    }));
+    await withHardCap(page.evaluate(async () => {
       const fonts = document.fonts;
       if (fonts?.ready) await fonts.ready;
     }).catch(() => {
-    });
-    await scrollToTriggerLazyLoad(page);
-    await waitForStableDOM(page, 1e4, 700);
-    await page.evaluate(async () => {
+    }));
+    await withHardCap(scrollToTriggerLazyLoad(page));
+    await withHardCap(waitForStableDOM(page, 4e3, 700));
+    await withHardCap(page.evaluate(async () => {
       const pending = Array.from(document.images).filter((img) => !img.complete);
       if (pending.length === 0) return;
       await Promise.race([
@@ -3431,10 +3446,10 @@ async function waitForScreenshotReady(page) {
         new Promise((resolve) => setTimeout(resolve, 4e3))
       ]);
     }).catch(() => {
-    });
-    await waitForStableDOM(page, 4e3, 500);
-    await page.waitForTimeout(250).catch(() => {
-    });
+    }));
+    await withHardCap(waitForStableDOM(page, 2e3, 500));
+    await withHardCap(page.waitForTimeout(250).catch(() => {
+    }));
   } catch {
   }
 }