@detiq/crawler 0.2.2 → 0.2.4

package/Dockerfile

@@ -5,6 +5,9 @@
 FROM mcr.microsoft.com/playwright:v1.55.0-jammy AS runner
 WORKDIR /app
 ENV NODE_ENV=production
+# Install browsers to a fixed path — base image ships browsers here and
+# we override after npm ci to match the exact installed playwright revision.
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 
 # Install Node.js 20 (Playwright base image ships with 18)
 RUN apt-get update && apt-get install -y curl && \
@@ -21,6 +24,11 @@ COPY packages/agents/package.json       packages/agents/
 COPY packages/crawler/package.json      packages/crawler/
 RUN npm ci
 
+# Re-install chromium after npm ci so the browser revision matches the
+# exact playwright version resolved by the lockfile (not the base image's
+# bundled version, which may differ if the lockfile was updated).
+RUN npx playwright install chromium
+
 # Source
 COPY tsconfig.base.json ./
 COPY packages/core       packages/core

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",

package/src/crawler.ts

@@ -978,11 +978,14 @@ export async function crawlProject(opts: {
     // Non-fatal — crawl proceeds without API endpoint capture
   }
   // Final fallback: if rawPage is still null, use the Stagehand page (events may not work)
-  if (!rawPage) rawPage = page;
+  if (!rawPage) {
+    rawPage = page;
+    logger.warn('[crawler] rawPage fell back to Stagehand proxy — CDP connectOverCDP failed; route() calls will be skipped');
+  }
 
   // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
   // Must use rawPage (real Playwright page) — Stagehand V3Page proxy does not implement route().
-  if (opts.httpBasicUsername && opts.httpBasicPassword) {
+  if (opts.httpBasicUsername && opts.httpBasicPassword && typeof rawPage?.route === 'function') {
     const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
     const appOrigin = new URL(opts.appUrl).origin;
     await rawPage.route('**/*', (route: any) => {
@@ -1002,7 +1005,7 @@ export async function crawlProject(opts: {
   // the crawler during long crawl sessions. The API verifies the secret before
   // skipping the limiter.
   const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
-  if (crawlerSecret) {
+  if (crawlerSecret && typeof rawPage?.route === 'function') {
     const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
     await rawPage.route(loginPattern, (route: any) => {
       route.continue({