@detiq/crawler 0.2.1 → 0.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/crawler.ts

@@ -3109,9 +3109,19 @@ export async function crawlProject(opts: {
       await onProgress?.(90, `App map saved · ${visited.size} screens, ${edges.length} flows wired`);
     }
 
+    // P5: Export session cookies BEFORE session health check so we can reuse them even if
+    // the main browser session is stale. The cookie jar stays valid for fresh browser contexts
+    // even when the CDP-connected Stagehand page shows a redirect to /login.
+    let mainStorageState: any;
+    try {
+      mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState();
+    } catch { /* non-fatal */ }
+
     // P5: Orphan page auth re-verify — confirm session is still alive before tail-end viewport pass.
     // The main BFS can span many minutes; the session may have expired by the time we reach here.
-    // If expired, attempt re-auth via Stagehand before proceeding. Only skip if re-auth also fails.
+    // If expired, attempt re-auth via Stagehand before proceeding.
+    // Creative resilience: even if the main browser is dead, we still run the viewport pass using
+    // the exported cookie jar — fresh browsers are often accepted by servers even after CDP session loss.
     let sessionAliveForViewports = true;
     try {
       await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 });
@@ -3130,19 +3140,24 @@ export async function crawlProject(opts: {
           ).catch(() => false);
         }
         if (reauthed) {
+          // Refresh cookie jar with newly authenticated session
+          try { mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState(); } catch { /* keep prior */ }
           logger.info('[P5] Re-auth succeeded — proceeding with viewport pass');
         } else {
           sessionAliveForViewports = false;
-          logger.warn({ landed: sessionCheckUrl }, '[P5] Re-auth failed — skipping mobile/tablet screenshots');
-          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
-            jobId: opts.jobId,
-            type: 'AUTH_FAILURE',
-            payload: {
-              url: opts.appUrl,
-              landed: sessionCheckUrl,
-              reason: 'Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings.',
-            },
-          }).catch(() => { });
+          const hasCookies = (mainStorageState?.cookies?.length ?? 0) > 0;
+          logger.warn({ landed: sessionCheckUrl, hasCookies }, '[P5] Re-auth failed — will attempt viewport pass with exported cookies');
+          if (!hasCookies) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId,
+              type: 'AUTH_FAILURE',
+              payload: {
+                url: opts.appUrl,
+                landed: sessionCheckUrl,
+                reason: 'Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings.',
+              },
+            }).catch(() => { });
+          }
         }
       } else {
         logger.info({ url: sessionCheckUrl }, '[P5] Session alive — proceeding with viewport pass');
@@ -3162,7 +3177,10 @@ export async function crawlProject(opts: {
     // (isMobile, hasTouch, viewport, deviceScaleFactor) on new contexts — the flags are
     // silently ignored, so screenshots come out at desktop dimensions regardless of profile.
     const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== 'DESKTOP');
-    if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
+    // Run the viewport pass even when the main session is dead, as long as we exported cookies —
+    // fresh browsers often authenticate successfully with the saved cookie jar.
+    const hasExportedAuth = (mainStorageState?.cookies?.length ?? 0) > 0;
+    if ((sessionAliveForViewports || hasExportedAuth) && extraProfiles.length > 0 && discovered.length > 0) {
       const { devices, chromium: pwChromium } = await import('playwright');
       const DEVICE_MAP: Record<string, any> = {
         // Override viewport heights: Playwright's device descriptors use viewport-minus-browser-chrome
@@ -3171,11 +3189,6 @@ export async function crawlProject(opts: {
         MOBILE_PORTRAIT: { ...devices['iPhone 15'], viewport: { width: 393, height: 852 } },
         TABLET: { ...devices['iPad Pro 11'], viewport: { width: 834, height: 1194 } },
       };
-      // Export session cookies from the main crawl so mobile browser is already authenticated
-      let mainStorageState: any;
-      try {
-        mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState();
-      } catch { /* non-fatal — mobile pass will proceed without auth */ }
 
       for (const profile of extraProfiles) {
         const deviceConfig = DEVICE_MAP[profile];
@@ -3224,13 +3237,19 @@ export async function crawlProject(opts: {
                 ).catch(() => { });
               }
             } catch (err) {
-              console.warn(`[crawler] ${profile} pass failed for ${disc.url}:`, err);
+              logger.warn({ profile, url: disc.url, err: String(err) }, `[P5] ${profile} screenshot failed — skipping screen`);
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                screenId: disc.screenId,
+                type: 'VIEWPORT_SCREENSHOT_FAILURE',
+                payload: { profile, url: disc.url, reason: String(err) },
+              }).catch(() => { });
             }
           }
           await mobilePage.close().catch(() => { });
           await mobileCtx.close().catch(() => { });
         } catch (profileErr) {
-          console.warn(`[crawler] ${profile} browser launch failed:`, profileErr);
+          logger.warn({ profile, err: String(profileErr) }, `[P5] ${profile} browser launch failed`);
         } finally {
           await mobileBrowser?.close().catch(() => { });
         }