@detiq/crawler 0.2.0 → 0.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/crawler.ts

@@ -853,24 +853,6 @@ export async function crawlProject(opts: {
     } catch { /* older engines without these entry types — leave stashed values at defaults */ }
   });
 
-  // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
-  // CDP-connected contexts cannot use browser.newContext({ httpCredentials }),
-  // so we intercept via page.route() instead.
-  if (opts.httpBasicUsername && opts.httpBasicPassword) {
-    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
-    const appOrigin = new URL(opts.appUrl).origin;
-    await page.route('**/*', (route: any) => {
-      const reqUrl = route.request().url();
-      if (reqUrl.startsWith(appOrigin)) {
-        route.continue({
-          headers: { ...route.request().headers(), 'Authorization': `Basic ${encoded}` },
-        });
-      } else {
-        route.continue();
-      }
-    });
-  }
-
   // ── API/Bearer token injection ────────────────────────────────────────────
   // If an authToken is provided, inject it as an HTTP header on every request
   // and store it in localStorage so client-side code can pick it up.
@@ -885,20 +867,6 @@ export async function crawlProject(opts: {
     }, opts.authToken);
   }
 
-  // ── Crawler login bypass: inject internal secret header on login requests ────
-  // Prevents the per-email rate limiter from blocking re-auth attempts made by
-  // the crawler during long crawl sessions. The API verifies the secret before
-  // skipping the limiter.
-  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
-  if (crawlerSecret) {
-    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
-    await page.route(loginPattern, (route: any) => {
-      route.continue({
-        headers: { ...route.request().headers(), 'x-zeta-crawler': crawlerSecret },
-      });
-    });
-  }
-
   // ── Web Vitals capture: CLS via buffered PerformanceObserver ─────────────────
   // CLS entries are only delivered to a PerformanceObserver; this init script
   // re-runs on every new document. LCP is captured per-page in the crawl loop.
@@ -1012,6 +980,37 @@ export async function crawlProject(opts: {
   // Final fallback: if rawPage is still null, use the Stagehand page (events may not work)
   if (!rawPage) rawPage = page;
 
+  // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
+  // Must use rawPage (real Playwright page) — Stagehand V3Page proxy does not implement route().
+  if (opts.httpBasicUsername && opts.httpBasicPassword) {
+    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
+    const appOrigin = new URL(opts.appUrl).origin;
+    await rawPage.route('**/*', (route: any) => {
+      const reqUrl = route.request().url();
+      if (reqUrl.startsWith(appOrigin)) {
+        route.continue({
+          headers: { ...route.request().headers(), 'Authorization': `Basic ${encoded}` },
+        });
+      } else {
+        route.continue();
+      }
+    });
+  }
+
+  // ── Crawler login bypass: inject internal secret header on login requests ────
+  // Prevents the per-email rate limiter from blocking re-auth attempts made by
+  // the crawler during long crawl sessions. The API verifies the secret before
+  // skipping the limiter.
+  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
+  if (crawlerSecret) {
+    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
+    await rawPage.route(loginPattern, (route: any) => {
+      route.continue({
+        headers: { ...route.request().headers(), 'x-zeta-crawler': crawlerSecret },
+      });
+    });
+  }
+
   try {
     // Step 1: Navigate and authenticate
     await onProgress?.(8, 'Loading app');
@@ -3110,9 +3109,19 @@ export async function crawlProject(opts: {
       await onProgress?.(90, `App map saved · ${visited.size} screens, ${edges.length} flows wired`);
     }
 
+    // P5: Export session cookies BEFORE session health check so we can reuse them even if
+    // the main browser session is stale. The cookie jar stays valid for fresh browser contexts
+    // even when the CDP-connected Stagehand page shows a redirect to /login.
+    let mainStorageState: any;
+    try {
+      mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState();
+    } catch { /* non-fatal */ }
+
     // P5: Orphan page auth re-verify — confirm session is still alive before tail-end viewport pass.
     // The main BFS can span many minutes; the session may have expired by the time we reach here.
-    // If expired, attempt re-auth via Stagehand before proceeding. Only skip if re-auth also fails.
+    // If expired, attempt re-auth via Stagehand before proceeding.
+    // Creative resilience: even if the main browser is dead, we still run the viewport pass using
+    // the exported cookie jar — fresh browsers are often accepted by servers even after CDP session loss.
     let sessionAliveForViewports = true;
     try {
       await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 });
@@ -3131,19 +3140,24 @@ export async function crawlProject(opts: {
           ).catch(() => false);
         }
         if (reauthed) {
+          // Refresh cookie jar with newly authenticated session
+          try { mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState(); } catch { /* keep prior */ }
           logger.info('[P5] Re-auth succeeded — proceeding with viewport pass');
         } else {
           sessionAliveForViewports = false;
-          logger.warn({ landed: sessionCheckUrl }, '[P5] Re-auth failed — skipping mobile/tablet screenshots');
-          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
-            jobId: opts.jobId,
-            type: 'AUTH_FAILURE',
-            payload: {
-              url: opts.appUrl,
-              landed: sessionCheckUrl,
-              reason: 'Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings.',
-            },
-          }).catch(() => { });
+          const hasCookies = (mainStorageState?.cookies?.length ?? 0) > 0;
+          logger.warn({ landed: sessionCheckUrl, hasCookies }, '[P5] Re-auth failed — will attempt viewport pass with exported cookies');
+          if (!hasCookies) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId,
+              type: 'AUTH_FAILURE',
+              payload: {
+                url: opts.appUrl,
+                landed: sessionCheckUrl,
+                reason: 'Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings.',
+              },
+            }).catch(() => { });
+          }
         }
       } else {
         logger.info({ url: sessionCheckUrl }, '[P5] Session alive — proceeding with viewport pass');
@@ -3163,7 +3177,10 @@ export async function crawlProject(opts: {
     // (isMobile, hasTouch, viewport, deviceScaleFactor) on new contexts — the flags are
     // silently ignored, so screenshots come out at desktop dimensions regardless of profile.
     const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== 'DESKTOP');
-    if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
+    // Run the viewport pass even when the main session is dead, as long as we exported cookies —
+    // fresh browsers often authenticate successfully with the saved cookie jar.
+    const hasExportedAuth = (mainStorageState?.cookies?.length ?? 0) > 0;
+    if ((sessionAliveForViewports || hasExportedAuth) && extraProfiles.length > 0 && discovered.length > 0) {
       const { devices, chromium: pwChromium } = await import('playwright');
       const DEVICE_MAP: Record<string, any> = {
         // Override viewport heights: Playwright's device descriptors use viewport-minus-browser-chrome
@@ -3172,11 +3189,6 @@ export async function crawlProject(opts: {
         MOBILE_PORTRAIT: { ...devices['iPhone 15'], viewport: { width: 393, height: 852 } },
         TABLET: { ...devices['iPad Pro 11'], viewport: { width: 834, height: 1194 } },
       };
-      // Export session cookies from the main crawl so mobile browser is already authenticated
-      let mainStorageState: any;
-      try {
-        mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState();
-      } catch { /* non-fatal — mobile pass will proceed without auth */ }
 
       for (const profile of extraProfiles) {
         const deviceConfig = DEVICE_MAP[profile];
@@ -3225,13 +3237,19 @@ export async function crawlProject(opts: {
                 ).catch(() => { });
               }
             } catch (err) {
-              console.warn(`[crawler] ${profile} pass failed for ${disc.url}:`, err);
+              logger.warn({ profile, url: disc.url, err: String(err) }, `[P5] ${profile} screenshot failed — skipping screen`);
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                screenId: disc.screenId,
+                type: 'VIEWPORT_SCREENSHOT_FAILURE',
+                payload: { profile, url: disc.url, reason: String(err) },
+              }).catch(() => { });
             }
           }
           await mobilePage.close().catch(() => { });
           await mobileCtx.close().catch(() => { });
         } catch (profileErr) {
-          console.warn(`[crawler] ${profile} browser launch failed:`, profileErr);
+          logger.warn({ profile, err: String(profileErr) }, `[P5] ${profile} browser launch failed`);
         } finally {
           await mobileBrowser?.close().catch(() => { });
         }