@detiq/crawler 0.2.6 → 0.2.9

package/dist/crawler.js

@@ -586,20 +586,6 @@ async function crawlProject(opts, onProgress) {
     } catch {
     }
   });
-  if (opts.httpBasicUsername && opts.httpBasicPassword) {
-    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString("base64");
-    const appOrigin = new URL(opts.appUrl).origin;
-    await page.route("**/*", (route) => {
-      const reqUrl = route.request().url();
-      if (reqUrl.startsWith(appOrigin)) {
-        route.continue({
-          headers: { ...route.request().headers(), "Authorization": `Basic ${encoded}` }
-        });
-      } else {
-        route.continue();
-      }
-    });
-  }
   if (opts.authToken) {
     const headerName = opts.authTokenHeader ?? "Authorization";
     const prefix = opts.authTokenPrefix ?? "Bearer ";
@@ -610,15 +596,6 @@ async function crawlProject(opts, onProgress) {
       localStorage.setItem("token", token);
     }, opts.authToken);
   }
-  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
-  if (crawlerSecret) {
-    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
-    await page.route(loginPattern, (route) => {
-      route.continue({
-        headers: { ...route.request().headers(), "x-zeta-crawler": crawlerSecret }
-      });
-    });
-  }
   await page.addInitScript(() => {
     try {
       window.__zetaCLS = 0;
@@ -698,7 +675,33 @@ async function crawlProject(opts, onProgress) {
     }
   } catch {
   }
-  if (!rawPage) rawPage = page;
+  if (!rawPage) {
+    rawPage = page;
+    logger.warn("[crawler] rawPage fell back to Stagehand proxy \u2014 CDP connectOverCDP failed; route() calls will be skipped");
+  }
+  if (opts.httpBasicUsername && opts.httpBasicPassword && typeof rawPage?.route === "function") {
+    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString("base64");
+    const appOrigin = new URL(opts.appUrl).origin;
+    await rawPage.route("**/*", (route) => {
+      const reqUrl = route.request().url();
+      if (reqUrl.startsWith(appOrigin)) {
+        route.continue({
+          headers: { ...route.request().headers(), "Authorization": `Basic ${encoded}` }
+        });
+      } else {
+        route.continue();
+      }
+    });
+  }
+  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
+  if (crawlerSecret && typeof rawPage?.route === "function") {
+    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
+    await rawPage.route(loginPattern, (route) => {
+      route.continue({
+        headers: { ...route.request().headers(), "x-zeta-crawler": crawlerSecret }
+      });
+    });
+  }
   try {
     await onProgress?.(8, "Loading app");
     await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(async () => {
@@ -2589,6 +2592,11 @@ ${snippet}`,
     if (visited.size >= 5) {
       await onProgress?.(90, `App map saved \xB7 ${visited.size} screens, ${edges.length} flows wired`);
     }
+    let mainStorageState;
+    try {
+      mainStorageState = await playwrightBrowser.contexts()[0]?.storageState();
+    } catch {
+    }
     let sessionAliveForViewports = true;
     try {
       await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 15e3 });
@@ -2609,20 +2617,27 @@ ${snippet}`,
           ).catch(() => false);
         }
         if (reauthed) {
+          try {
+            mainStorageState = await playwrightBrowser.contexts()[0]?.storageState();
+          } catch {
+          }
           logger.info("[P5] Re-auth succeeded \u2014 proceeding with viewport pass");
         } else {
           sessionAliveForViewports = false;
-          logger.warn({ landed: sessionCheckUrl }, "[P5] Re-auth failed \u2014 skipping mobile/tablet screenshots");
-          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
-            jobId: opts.jobId,
-            type: "AUTH_FAILURE",
-            payload: {
-              url: opts.appUrl,
-              landed: sessionCheckUrl,
-              reason: "Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings."
-            }
-          }).catch(() => {
-          });
+          const hasCookies = (mainStorageState?.cookies?.length ?? 0) > 0;
+          logger.warn({ landed: sessionCheckUrl, hasCookies }, "[P5] Re-auth failed \u2014 will attempt viewport pass with exported cookies");
+          if (!hasCookies) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              jobId: opts.jobId,
+              type: "AUTH_FAILURE",
+              payload: {
+                url: opts.appUrl,
+                landed: sessionCheckUrl,
+                reason: "Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings."
+              }
+            }).catch(() => {
+            });
+          }
         }
       } else {
         logger.info({ url: sessionCheckUrl }, "[P5] Session alive \u2014 proceeding with viewport pass");
@@ -2631,7 +2646,8 @@ ${snippet}`,
       logger.warn({ err: String(err) }, "[P5] Session health check navigation failed \u2014 proceeding with viewport pass anyway");
     }
     const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== "DESKTOP");
-    if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
+    const hasExportedAuth = (mainStorageState?.cookies?.length ?? 0) > 0;
+    if ((sessionAliveForViewports || hasExportedAuth) && extraProfiles.length > 0 && discovered.length > 0) {
       const { devices, chromium: pwChromium } = await import("playwright");
       const DEVICE_MAP = {
         // Override viewport heights: Playwright's device descriptors use viewport-minus-browser-chrome
@@ -2640,11 +2656,6 @@ ${snippet}`,
         MOBILE_PORTRAIT: { ...devices["iPhone 15"], viewport: { width: 393, height: 852 } },
         TABLET: { ...devices["iPad Pro 11"], viewport: { width: 834, height: 1194 } }
       };
-      let mainStorageState;
-      try {
-        mainStorageState = await playwrightBrowser.contexts()[0]?.storageState();
-      } catch {
-      }
       for (const profile of extraProfiles) {
         const deviceConfig = DEVICE_MAP[profile];
         if (!deviceConfig) continue;
@@ -2702,7 +2713,14 @@ ${snippet}`,
                 });
               }
             } catch (err) {
-              console.warn(`[crawler] ${profile} pass failed for ${disc.url}:`, err);
+              logger.warn({ profile, url: disc.url, err: String(err) }, `[P5] ${profile} screenshot failed \u2014 skipping screen`);
+              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+                jobId: opts.jobId,
+                screenId: disc.screenId,
+                type: "VIEWPORT_SCREENSHOT_FAILURE",
+                payload: { profile, url: disc.url, reason: String(err) }
+              }).catch(() => {
+              });
             }
           }
           await mobilePage.close().catch(() => {
@@ -2710,7 +2728,7 @@ ${snippet}`,
           await mobileCtx.close().catch(() => {
           });
         } catch (profileErr) {
-          console.warn(`[crawler] ${profile} browser launch failed:`, profileErr);
+          logger.warn({ profile, err: String(profileErr) }, `[P5] ${profile} browser launch failed`);
         } finally {
           await mobileBrowser?.close().catch(() => {
           });