@detiq/crawler 0.2.0 → 0.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/crawler.ts

@@ -853,24 +853,6 @@ export async function crawlProject(opts: {
     } catch { /* older engines without these entry types — leave stashed values at defaults */ }
   });
 
-  // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
-  // CDP-connected contexts cannot use browser.newContext({ httpCredentials }),
-  // so we intercept via page.route() instead.
-  if (opts.httpBasicUsername && opts.httpBasicPassword) {
-    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
-    const appOrigin = new URL(opts.appUrl).origin;
-    await page.route('**/*', (route: any) => {
-      const reqUrl = route.request().url();
-      if (reqUrl.startsWith(appOrigin)) {
-        route.continue({
-          headers: { ...route.request().headers(), 'Authorization': `Basic ${encoded}` },
-        });
-      } else {
-        route.continue();
-      }
-    });
-  }
-
   // ── API/Bearer token injection ────────────────────────────────────────────
   // If an authToken is provided, inject it as an HTTP header on every request
   // and store it in localStorage so client-side code can pick it up.
@@ -885,20 +867,6 @@ export async function crawlProject(opts: {
     }, opts.authToken);
   }
 
-  // ── Crawler login bypass: inject internal secret header on login requests ────
-  // Prevents the per-email rate limiter from blocking re-auth attempts made by
-  // the crawler during long crawl sessions. The API verifies the secret before
-  // skipping the limiter.
-  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
-  if (crawlerSecret) {
-    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
-    await page.route(loginPattern, (route: any) => {
-      route.continue({
-        headers: { ...route.request().headers(), 'x-zeta-crawler': crawlerSecret },
-      });
-    });
-  }
-
   // ── Web Vitals capture: CLS via buffered PerformanceObserver ─────────────────
   // CLS entries are only delivered to a PerformanceObserver; this init script
   // re-runs on every new document. LCP is captured per-page in the crawl loop.
@@ -1012,6 +980,37 @@ export async function crawlProject(opts: {
   // Final fallback: if rawPage is still null, use the Stagehand page (events may not work)
   if (!rawPage) rawPage = page;
 
+  // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
+  // Must use rawPage (real Playwright page) — Stagehand V3Page proxy does not implement route().
+  if (opts.httpBasicUsername && opts.httpBasicPassword) {
+    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
+    const appOrigin = new URL(opts.appUrl).origin;
+    await rawPage.route('**/*', (route: any) => {
+      const reqUrl = route.request().url();
+      if (reqUrl.startsWith(appOrigin)) {
+        route.continue({
+          headers: { ...route.request().headers(), 'Authorization': `Basic ${encoded}` },
+        });
+      } else {
+        route.continue();
+      }
+    });
+  }
+
+  // ── Crawler login bypass: inject internal secret header on login requests ────
+  // Prevents the per-email rate limiter from blocking re-auth attempts made by
+  // the crawler during long crawl sessions. The API verifies the secret before
+  // skipping the limiter.
+  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
+  if (crawlerSecret) {
+    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
+    await rawPage.route(loginPattern, (route: any) => {
+      route.continue({
+        headers: { ...route.request().headers(), 'x-zeta-crawler': crawlerSecret },
+      });
+    });
+  }
+
   try {
     // Step 1: Navigate and authenticate
     await onProgress?.(8, 'Loading app');