@detiq/crawler 0.1.8 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.1.8",
+  "version": "0.2.0",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/crawler.ts

@@ -141,6 +141,7 @@ type AuthDiscoveryOptions = {
   projectId: string;
   jobId?: string;
   screenshotDir?: string;
+  storageConfig?: any;
 };
 
 const AUTH_DISCOVERY_MAX_SCREENS = 36;
@@ -403,6 +404,8 @@ export async function crawlProject(opts: {
   /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
   storageState?: string;
   screenshotDir?: string;
+  /** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */
+  storageConfig?: { provider: string; bucket: string; region?: string; accountId?: string; endpointUrl?: string; accessKeyId?: string; secretAccessKey?: string; prefix?: string; publicUrl?: string } | null;
   /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
   discoveryMode?: boolean;
   maxScreens?: number;
@@ -504,6 +507,17 @@ export async function crawlProject(opts: {
 }, onProgress?: (pct: number, step: string) => Promise<void>) {
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
 
+  // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
+  const uploadToCloud = async (localPath: string, key: string): Promise<string | undefined> => {
+    if (!opts.storageConfig) return undefined;
+    try {
+      const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
+      return await uploadScreenshotToTenantStorage(opts.storageConfig as any, localPath, key);
+    } catch {
+      return undefined;
+    }
+  };
+
   const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === 'production';
 
   // Use tenant LLM config if available; fall back to env vars (Anthropic only for Stagehand).
@@ -871,6 +885,20 @@ export async function crawlProject(opts: {
     }, opts.authToken);
   }
 
+  // ── Crawler login bypass: inject internal secret header on login requests ────
+  // Prevents the per-email rate limiter from blocking re-auth attempts made by
+  // the crawler during long crawl sessions. The API verifies the secret before
+  // skipping the limiter.
+  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
+  if (crawlerSecret) {
+    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
+    await page.route(loginPattern, (route: any) => {
+      route.continue({
+        headers: { ...route.request().headers(), 'x-zeta-crawler': crawlerSecret },
+      });
+    });
+  }
+
   // ── Web Vitals capture: CLS via buffered PerformanceObserver ─────────────────
   // CLS entries are only delivered to a PerformanceObserver; this init script
   // re-runs on every new document. LCP is captured per-page in the crawl loop.
@@ -1520,7 +1548,13 @@ export async function crawlProject(opts: {
     let totalAttemptsThisSegment = 0;
     const maxAttempts = Math.max(segmentSize * 4, 200);
 
+    let pendingSessionRestore: (() => Promise<void>) | null = null;
+
     while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+      if (pendingSessionRestore) {
+        await pendingSessionRestore();
+        pendingSessionRestore = null;
+      }
       if (++totalAttemptsThisSegment > maxAttempts) {
         logger.warn({ maxAttempts, visited: visited.size, queued: queue.length }, '[crawler] attempt cap reached — stopping to prevent URL explosion');
         break;
@@ -1610,6 +1644,41 @@ export async function crawlProject(opts: {
             }
           };
           rawPage.on('response', onRedirectResponse);
+
+          // Session Isolation: Clear session before crawling public auth routes
+          const AUTH_PATH_REGEX = /\/(login|register|signup|sign-in|sign-up|forgot-password|reset-password)([-/]|$)/i;
+          const isAuthRoute = AUTH_PATH_REGEX.test(new URL(url).pathname);
+          const browserCtx = playwrightBrowser?.contexts()[0];
+          
+          if (isAuthRoute && browserCtx) {
+            const currentState = await browserCtx.storageState();
+            const hasSession = (currentState.cookies?.length ?? 0) > 0 ||
+              currentState.origins?.some((o: any) => (o.localStorage?.length ?? 0) > 0);
+            if (hasSession) {
+              pendingSessionRestore = async () => {
+                const ctx = playwrightBrowser?.contexts()[0];
+                if (ctx) {
+                  await ctx.addCookies(currentState.cookies);
+                  await rawPage.evaluate((origins: any) => {
+                    for (const entry of (origins ?? [])) {
+                      if (window.location.origin === entry.origin) {
+                        for (const item of entry.localStorage) {
+                          try { localStorage.setItem(item.name, item.value); } catch {}
+                        }
+                      }
+                    }
+                  }, currentState.origins).catch(() => {});
+                  logger.info('[crawler] restored session state after public auth route');
+                }
+              };
+              await browserCtx.clearCookies();
+              await rawPage.evaluate(() => {
+                try { localStorage.clear(); sessionStorage.clear(); } catch {}
+              }).catch(() => {});
+              logger.info({ url }, '[crawler] temporarily cleared session for public auth route');
+            }
+          }
+
           await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
             logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
           });
@@ -2163,7 +2232,10 @@ export async function crawlProject(opts: {
         // Parallelize: screenshot + element extraction + link discovery + AT capture (all work on current page state)
         const [, elements, nextLinks, atElements] = await Promise.all([
           page.screenshot({ path: shotPath, fullPage: true })
-            .then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP'))
+            .then(async () => {
+              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+              return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl);
+            })
             .catch((err: any) => console.error('[crawler] page.screenshot failed:', err)),
           extractElements(stagehand, page, elementLLM ?? undefined).catch((err: any) => {
             console.error('[crawler] extractElements failed:', err);
@@ -2791,6 +2863,11 @@ export async function crawlProject(opts: {
       }
     }
 
+    if (pendingSessionRestore) {
+      await pendingSessionRestore();
+      pendingSessionRestore = null;
+    }
+
     // P1: URL Pattern LLM Completion — post-crawl LLM prediction of undiscovered URLs.
     // Single LLM call predicts missing routes from discovered URL patterns, HEAD-validates each.
     // Saves URL_CANDIDATES evidence so the ActivityDrawer shows what was found.
@@ -3135,7 +3212,8 @@ export async function crawlProject(opts: {
                 mobilePage.screenshot({ path: shotPath, fullPage: true }),
                 domFallbackElements(mobilePage),
               ]);
-              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version!, profile);
+              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version!, profile, cloudUrl);
               AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, disc.screenId, profile, shotPath, gitInfo.branch, gitInfo.commit || undefined).catch(() => { });
               // Save viewport-specific element records (bounding rects differ per viewport)
               for (const el of vpElements) {
@@ -4213,7 +4291,14 @@ async function captureAuthDiscoveryScreen(
   await page.screenshot({ path: shotPath, fullPage: true }).catch(() => { });
   const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === 'entry' ? 'ENTRY_POINT' : 'SEED_URL').catch(() => null);
   if (!screen) return { captured: false, finalUrl, flowType };
-  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP').catch(() => { });
+  let authCloudUrl: string | undefined;
+  if (opts.storageConfig) {
+    try {
+      const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
+      authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig as any, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+    } catch { /* non-fatal */ }
+  }
+  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,

package/src/job-queue.ts

@@ -621,8 +621,10 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
         }
       : undefined;
 
+    const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
     const result = await crawlProject({
       ...config, credentials: runtimeCreds, abortSignal: controller.signal, jobId,
+      storageConfig: storageConfigForCrawl,
       onCheckpoint: async (visitedUrls, pendingQueue) => {
         jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
       },

package/types/crawler.d.ts

@@ -41,6 +41,18 @@ export declare function crawlProject(opts: {
     /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
     storageState?: string;
     screenshotDir?: string;
+    /** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */
+    storageConfig?: {
+        provider: string;
+        bucket: string;
+        region?: string;
+        accountId?: string;
+        endpointUrl?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+        prefix?: string;
+        publicUrl?: string;
+    } | null;
     /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
     discoveryMode?: boolean;
     maxScreens?: number;