@detiq/crawler 0.1.5 → 0.1.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/ai-login.ts

@@ -141,7 +141,20 @@ export async function loginWithStagehand(
     await logStep('fill_password', async () => {
       const observations = await stagehand.observe('Find the password input field');
       const selector = observations[0]?.selector || 'input[type="password"]';
-      await pg.locator(selector).fill(creds.password);
+      try {
+        await pg.locator(selector).fill(creds.password);
+      } catch {
+        // Stagehand locator failed (e.g. form re-rendered, proxy can't locate element).
+        // Fall back to direct DOM fill via evaluate so the password never touches the LLM.
+        await pg.evaluate((pwd: string) => {
+          const input = document.querySelector('input[type="password"]') as HTMLInputElement | null;
+          if (!input) throw new Error('No password field found on page');
+          const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;
+          setter?.call(input, pwd);
+          input.dispatchEvent(new Event('input', { bubbles: true }));
+          input.dispatchEvent(new Event('change', { bubbles: true }));
+        }, creds.password);
+      }
     });
     await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));
     await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));

package/src/crawler.ts

@@ -1059,9 +1059,13 @@ export async function crawlProject(opts: {
           }, state.origins);
         }
 
-        // Reload to apply cookies + trigger localStorage restore initScript
+        // Navigate to appUrl with cookies now set.
+        // We MUST navigate to appUrl (not reload current page) because the initial
+        // goto ran without cookies and was redirected to /login. Reloading /login stays
+        // at /login even with a valid cookie (public path, no auth redirect). Navigating
+        // to appUrl lets server middleware see the cookie and route to the authenticated app.
         await onProgress?.(12, 'Restoring session');
-        await rawPage.reload({ waitUntil: 'domcontentloaded' });
+        await rawPage.goto(opts.appUrl, { waitUntil: 'domcontentloaded' }).catch(() => { });
         await waitForStableDOM(page);
         // SPA root pages redirect client-side (useEffect → router.replace). Wait for it.
         if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, '')) {
@@ -1303,7 +1307,8 @@ export async function crawlProject(opts: {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
-    let lastEmittedPct = 20; // clamp so progress never decreases even when queue grows
+    let lastEmittedPct = 5; // clamp so progress never decreases even when queue grows
+    const crawlLoopStart = Date.now();
     const startUrl = page.url();
     // Resume from previous segment's pending queue if provided
     const resumeUrls = opts.resumeUrls ?? [];
@@ -2135,7 +2140,13 @@ export async function crawlProject(opts: {
 
         // Emit progress now — navigation is done and the page is stable.
         // This fires before the 30-90s parallel screenshot+AI phase, preventing a silent gap in the UI.
-        await onProgress?.(20, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
+        // Phase 1 (link discovery): 5–20% over first 60s of wall-clock time so users see movement
+        // before visited.size grows enough to drive the 20–80% phase.
+        const elapsedDiscoverySec = (Date.now() - crawlLoopStart) / 1000;
+        const discoveryPct = Math.round(5 + Math.min(15, (elapsedDiscoverySec / 60) * 15));
+        const preCapturePct = Math.max(lastEmittedPct, discoveryPct);
+        lastEmittedPct = preCapturePct;
+        await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
 
         // P3a: Async content detection — get raw DOM element count before AI extraction.
         // The AI extraction takes 30-90s; any async-loaded content will be visible after it completes.
@@ -2145,6 +2156,7 @@ export async function crawlProject(opts: {
         ).catch(() => 0);
 
         await waitForScreenshotReady(page);
+        await onProgress?.(lastEmittedPct, `Capturing ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
 
         // Parallelize: screenshot + element extraction + link discovery + AT capture (all work on current page state)
         const [, elements, nextLinks, atElements] = await Promise.all([
@@ -2644,9 +2656,10 @@ export async function crawlProject(opts: {
 
         discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
         // Progress: 20% baseline + up to 60% across discovered screens.
-        // maxSeenTotal is the denominator high-water mark; lastEmittedPct prevents
-        // the displayed % from going backwards when new link discoveries grow the queue.
-        maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
+        // Cap maxSeenTotal at maxScreens * 3 so unbounded link discovery does not
+        // collapse the denominator and freeze progress in the 20-22% band.
+        const effectiveMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
+        maxSeenTotal = Math.max(maxSeenTotal, effectiveMax);
         const rawPct = 20 + Math.round(Math.min(60, (visited.size / Math.max(1, maxSeenTotal)) * 60));
         const clampedPct = Math.max(lastEmittedPct, rawPct);
         lastEmittedPct = clampedPct;
@@ -4018,24 +4031,27 @@ async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400):
 }
 
 async function waitForScreenshotReady(page: any): Promise<void> {
+  const hardDeadline = Date.now() + 20_000;
+  const withHardCap = <T>(p: Promise<T>): Promise<T | void> =>
+    Promise.race([p, new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
   try {
-    await page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { });
-    await page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { });
-    await page.evaluate(() => {
+    await withHardCap(page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { }));
+    await withHardCap(page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { }));
+    await withHardCap(page.evaluate(() => {
       if (document.readyState === 'complete') return true;
       return new Promise<boolean>((resolve) => {
         const done = () => resolve(true);
         window.addEventListener('load', done, { once: true });
         setTimeout(done, 4_000);
       });
-    }).catch(() => { });
-    await page.evaluate(async () => {
+    }).catch(() => { }));
+    await withHardCap(page.evaluate(async () => {
       const fonts = (document as any).fonts;
       if (fonts?.ready) await fonts.ready;
-    }).catch(() => { });
-    await scrollToTriggerLazyLoad(page);
-    await waitForStableDOM(page, 10_000, 700);
-    await page.evaluate(async () => {
+    }).catch(() => { }));
+    await withHardCap(scrollToTriggerLazyLoad(page));
+    await withHardCap(waitForStableDOM(page, 4_000, 700));
+    await withHardCap(page.evaluate(async () => {
       const pending = Array.from(document.images).filter((img) => !img.complete);
       if (pending.length === 0) return;
       await Promise.race([
@@ -4045,9 +4061,9 @@ async function waitForScreenshotReady(page: any): Promise<void> {
         }))),
         new Promise<void>((resolve) => setTimeout(resolve, 4_000)),
       ]);
-    }).catch(() => { });
-    await waitForStableDOM(page, 4_000, 500);
-    await page.waitForTimeout(250).catch(() => { });
+    }).catch(() => { }));
+    await withHardCap(waitForStableDOM(page, 2_000, 500));
+    await withHardCap(page.waitForTimeout(250).catch(() => { }));
   } catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
 }
 

package/src/skyvern.ts

@@ -67,9 +67,6 @@ export async function skyvernNavigate(
     return { screenshotUrls: [], visitedUrls: [], status: 'failed' };
   }
 
-  const { assertResolvesToPublicAddress } = await import('@detiq/core');
-  await assertResolvesToPublicAddress(apiUrl);
-
   const visitedUrls: string[] = [];
   const screenshotUrls: string[] = [];
   const skyvernSteps: SkyvernStep[] = [];
@@ -164,9 +161,6 @@ export async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernC
     return 'failed';
   }
 
-  const { assertResolvesToPublicAddress } = await import('@detiq/core');
-  await assertResolvesToPublicAddress(apiUrl);
-
   try {
     const res = await fetch(`${apiUrl}/api/v1/tasks`, {
       method: 'POST',