@detiq/crawler 0.1.5 → 0.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/crawler.ts

@@ -1303,7 +1303,8 @@ export async function crawlProject(opts: {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
-    let lastEmittedPct = 20; // clamp so progress never decreases even when queue grows
+    let lastEmittedPct = 5; // clamp so progress never decreases even when queue grows
+    const crawlLoopStart = Date.now();
     const startUrl = page.url();
     // Resume from previous segment's pending queue if provided
     const resumeUrls = opts.resumeUrls ?? [];
@@ -2135,7 +2136,13 @@ export async function crawlProject(opts: {
 
         // Emit progress now — navigation is done and the page is stable.
         // This fires before the 30-90s parallel screenshot+AI phase, preventing a silent gap in the UI.
-        await onProgress?.(20, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
+        // Phase 1 (link discovery): 5–20% over first 60s of wall-clock time so users see movement
+        // before visited.size grows enough to drive the 20–80% phase.
+        const elapsedDiscoverySec = (Date.now() - crawlLoopStart) / 1000;
+        const discoveryPct = Math.round(5 + Math.min(15, (elapsedDiscoverySec / 60) * 15));
+        const preCapturePct = Math.max(lastEmittedPct, discoveryPct);
+        lastEmittedPct = preCapturePct;
+        await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
 
         // P3a: Async content detection — get raw DOM element count before AI extraction.
         // The AI extraction takes 30-90s; any async-loaded content will be visible after it completes.
@@ -2145,6 +2152,7 @@ export async function crawlProject(opts: {
         ).catch(() => 0);
 
         await waitForScreenshotReady(page);
+        await onProgress?.(lastEmittedPct, `Capturing ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
 
         // Parallelize: screenshot + element extraction + link discovery + AT capture (all work on current page state)
         const [, elements, nextLinks, atElements] = await Promise.all([
@@ -2644,9 +2652,10 @@ export async function crawlProject(opts: {
 
         discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
         // Progress: 20% baseline + up to 60% across discovered screens.
-        // maxSeenTotal is the denominator high-water mark; lastEmittedPct prevents
-        // the displayed % from going backwards when new link discoveries grow the queue.
-        maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
+        // Cap maxSeenTotal at maxScreens * 3 so unbounded link discovery does not
+        // collapse the denominator and freeze progress in the 20-22% band.
+        const effectiveMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+        maxSeenTotal = Math.max(maxSeenTotal, effectiveMax);
         const rawPct = 20 + Math.round(Math.min(60, (visited.size / Math.max(1, maxSeenTotal)) * 60));
         const clampedPct = Math.max(lastEmittedPct, rawPct);
         lastEmittedPct = clampedPct;
@@ -4018,24 +4027,27 @@ async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400):
 }
 
 async function waitForScreenshotReady(page: any): Promise<void> {
+  const hardDeadline = Date.now() + 20_000;
+  const withHardCap = <T>(p: Promise<T>): Promise<T | void> =>
+    Promise.race([p, new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
   try {
-    await page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { });
-    await page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { });
-    await page.evaluate(() => {
+    await withHardCap(page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { }));
+    await withHardCap(page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { }));
+    await withHardCap(page.evaluate(() => {
       if (document.readyState === 'complete') return true;
       return new Promise<boolean>((resolve) => {
         const done = () => resolve(true);
         window.addEventListener('load', done, { once: true });
         setTimeout(done, 4_000);
       });
-    }).catch(() => { });
-    await page.evaluate(async () => {
+    }).catch(() => { }));
+    await withHardCap(page.evaluate(async () => {
       const fonts = (document as any).fonts;
       if (fonts?.ready) await fonts.ready;
-    }).catch(() => { });
-    await scrollToTriggerLazyLoad(page);
-    await waitForStableDOM(page, 10_000, 700);
-    await page.evaluate(async () => {
+    }).catch(() => { }));
+    await withHardCap(scrollToTriggerLazyLoad(page));
+    await withHardCap(waitForStableDOM(page, 4_000, 700));
+    await withHardCap(page.evaluate(async () => {
       const pending = Array.from(document.images).filter((img) => !img.complete);
       if (pending.length === 0) return;
       await Promise.race([
@@ -4045,9 +4057,9 @@ async function waitForScreenshotReady(page: any): Promise<void> {
         }))),
         new Promise<void>((resolve) => setTimeout(resolve, 4_000)),
       ]);
-    }).catch(() => { });
-    await waitForStableDOM(page, 4_000, 500);
-    await page.waitForTimeout(250).catch(() => { });
+    }).catch(() => { }));
+    await withHardCap(waitForStableDOM(page, 2_000, 500));
+    await withHardCap(page.waitForTimeout(250).catch(() => { }));
   } catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
 }