@detiq/crawler 0.1.8 → 0.2.0

package/dist/crawler.js

@@ -314,6 +314,15 @@ const TwoFASchema = z.object({
 });
 async function crawlProject(opts, onProgress) {
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
+  const uploadToCloud = async (localPath, key) => {
+    if (!opts.storageConfig) return void 0;
+    try {
+      const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
+      return await uploadScreenshotToTenantStorage(opts.storageConfig, localPath, key);
+    } catch {
+      return void 0;
+    }
+  };
   const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === "production";
   const toStagehandModel = (model, provider) => model.includes("/") ? model : `${provider}/${model}`;
   let stagehandModel = toStagehandModel(config.llmModel, "openai");
@@ -601,6 +610,15 @@ async function crawlProject(opts, onProgress) {
       localStorage.setItem("token", token);
     }, opts.authToken);
   }
+  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
+  if (crawlerSecret) {
+    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
+    await page.route(loginPattern, (route) => {
+      route.continue({
+        headers: { ...route.request().headers(), "x-zeta-crawler": crawlerSecret }
+      });
+    });
+  }
   await page.addInitScript(() => {
     try {
       window.__zetaCLS = 0;
@@ -1179,7 +1197,12 @@ async function crawlProject(opts, onProgress) {
     let aiRateLimitWarned = false;
     let totalAttemptsThisSegment = 0;
     const maxAttempts = Math.max(segmentSize * 4, 200);
+    let pendingSessionRestore = null;
     while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+      if (pendingSessionRestore) {
+        await pendingSessionRestore();
+        pendingSessionRestore = null;
+      }
       if (++totalAttemptsThisSegment > maxAttempts) {
         logger.warn({ maxAttempts, visited: visited.size, queued: queue.length }, "[crawler] attempt cap reached \u2014 stopping to prevent URL explosion");
         break;
@@ -1251,6 +1274,45 @@ async function crawlProject(opts, onProgress) {
             }
           };
           rawPage.on("response", onRedirectResponse);
+          const AUTH_PATH_REGEX = /\/(login|register|signup|sign-in|sign-up|forgot-password|reset-password)([-/]|$)/i;
+          const isAuthRoute = AUTH_PATH_REGEX.test(new URL(url).pathname);
+          const browserCtx = playwrightBrowser?.contexts()[0];
+          if (isAuthRoute && browserCtx) {
+            const currentState = await browserCtx.storageState();
+            const hasSession = (currentState.cookies?.length ?? 0) > 0 || currentState.origins?.some((o) => (o.localStorage?.length ?? 0) > 0);
+            if (hasSession) {
+              pendingSessionRestore = async () => {
+                const ctx = playwrightBrowser?.contexts()[0];
+                if (ctx) {
+                  await ctx.addCookies(currentState.cookies);
+                  await rawPage.evaluate((origins) => {
+                    for (const entry of origins ?? []) {
+                      if (window.location.origin === entry.origin) {
+                        for (const item of entry.localStorage) {
+                          try {
+                            localStorage.setItem(item.name, item.value);
+                          } catch {
+                          }
+                        }
+                      }
+                    }
+                  }, currentState.origins).catch(() => {
+                  });
+                  logger.info("[crawler] restored session state after public auth route");
+                }
+              };
+              await browserCtx.clearCookies();
+              await rawPage.evaluate(() => {
+                try {
+                  localStorage.clear();
+                  sessionStorage.clear();
+                } catch {
+                }
+              }).catch(() => {
+              });
+              logger.info({ url }, "[crawler] temporarily cleared session for public auth route");
+            }
+          }
           await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch((navErr) => {
             logger.warn({ url, err: navErr?.message ?? String(navErr) }, "[crawler] navigation failed \u2014 skipping");
           });
@@ -1733,7 +1795,10 @@ async function crawlProject(opts, onProgress) {
           }
         })()}`);
         const [, elements, nextLinks, atElements] = await Promise.all([
-          page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
+          page.screenshot({ path: shotPath, fullPage: true }).then(async () => {
+            const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+            return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl);
+          }).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
           extractElements(stagehand, page, elementLLM ?? void 0).catch((err) => {
             console.error("[crawler] extractElements failed:", err);
             return [];
@@ -2291,6 +2356,10 @@ ${snippet}`,
         }
       }
     }
+    if (pendingSessionRestore) {
+      await pendingSessionRestore();
+      pendingSessionRestore = null;
+    }
     try {
       const { runUrlPatternCompletion } = await import("./url-pattern-completer.js");
       const llmCfgForPatterns = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => null);
@@ -2610,7 +2679,8 @@ ${snippet}`,
                 mobilePage.screenshot({ path: shotPath, fullPage: true }),
                 domFallbackElements(mobilePage)
               ]);
-              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version, profile);
+              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version, profile, cloudUrl);
               AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, disc.screenId, profile, shotPath, gitInfo.branch, gitInfo.commit || void 0).catch(() => {
               });
               for (const el of vpElements) {
@@ -3572,7 +3642,15 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
   });
   const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === "entry" ? "ENTRY_POINT" : "SEED_URL").catch(() => null);
   if (!screen) return { captured: false, finalUrl, flowType };
-  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP").catch(() => {
+  let authCloudUrl;
+  if (opts.storageConfig) {
+    try {
+      const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
+      authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+    } catch {
+    }
+  }
+  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", authCloudUrl).catch(() => {
   });
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,