@detiq/crawler 1.1.20 → 1.1.22

package/Dockerfile

@@ -36,6 +36,11 @@ COPY packages/vscode-extension/package.json packages/vscode-extension/
 COPY packages/web/package.json          packages/web/
 RUN npm ci --include=dev
 
+# Patch Stagehand's act() schema to add `additionalProperties: false` on the
+# inner anyOf[0] object. OpenAI strict mode (v1/responses) requires it on every
+# nested object; Stagehand ≤3.6 omits it, causing a 400 "invalid_json_schema".
+RUN node -e "const fs=require('fs');['packages/crawler/node_modules/@browserbasehq/stagehand/dist/cjs/lib/inference.js','packages/crawler/node_modules/@browserbasehq/stagehand/dist/esm/lib/inference.js'].forEach(f=>{if(!fs.existsSync(f)){console.log('skip',f);return;}let c=fs.readFileSync(f,'utf8');if(c.includes('.strict()')){console.log('already patched',f);return;}c=c.replace(/\\.nullable\\(\\)\\n(\\s+)\\.describe\\(\"The element to act on/g,'\.strict()\n\$1.nullable()\n\$1.describe(\"The element to act on');fs.writeFileSync(f,c);console.log('patched',f);});"
+
 # Re-install chromium after npm ci so the browser revision matches the
 # exact playwright version resolved by the lockfile (not the base image's
 # bundled version, which may differ if the lockfile was updated).

package/dist/crawler.js

@@ -73,7 +73,7 @@ const _crawlLLMCache = /* @__PURE__ */ new Map();
 async function getCachedCrawlLLM(tenantId) {
   if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
   const cfg = await getTaskAgentLLMConfig(tenantId, "crawl").catch(() => null);
-  _crawlLLMCache.set(tenantId, cfg);
+  if (cfg) _crawlLLMCache.set(tenantId, cfg);
   return cfg;
 }
 function getGitInfo() {
@@ -88,7 +88,7 @@ function getGitInfo() {
 const ALLOWED_NUMERIC_PATHS = /* @__PURE__ */ new Set(["/2fa", "/404", "/500", "/403", "/401"]);
 const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
-const NUMERIC_ID_PATH_SEGMENT_RE = /^\d[\d-]*$/;
+const NUMERIC_ID_PATH_SEGMENT_RE = /^\d{4,}$/;
 const COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE = /^\d+-\d+$/;
 function normalizePathForNumericGuard(pathname) {
   const normalized = pathname.replace(/\/+$/, "") || "/";
@@ -374,6 +374,7 @@ const TwoFASchema = z.object({
   has2FA: z.boolean(),
   fieldLabel: z.string().optional()
 });
+const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
 async function crawlProject(opts, onProgress) {
   const tuning = await getCrawlTuning();
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
@@ -1597,6 +1598,7 @@ async function crawlProject(opts, onProgress) {
     };
     const edges = [];
     const pendingTransitions = [];
+    const pendingSummaryWrites = [];
     const discovered = [];
     const performanceMetrics = [];
     const seenDomHashes = new Set(opts.knownDomHashes ?? []);
@@ -1681,7 +1683,6 @@ async function crawlProject(opts, onProgress) {
           const name = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
           visited.set(canonUrl, screen.id);
-          const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
           const lwBodyText = await workerPage.evaluate(() => document.body?.innerText ?? "").catch(() => "");
           if (INLINE_AUTH_WALL_RE.test(lwBodyText)) {
             logger.warn({ url, screenId: screen.id }, "[lite-worker] Inline session-expired content detected \u2014 marking requiresAuth");
@@ -2875,9 +2876,9 @@ async function crawlProject(opts, onProgress) {
             })();
           }
         }
-        const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
+        const INLINE_AUTH_WALL_RE2 = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
         const pageBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
-        if (INLINE_AUTH_WALL_RE.test(pageBodyText)) {
+        if (INLINE_AUTH_WALL_RE2.test(pageBodyText)) {
           logger.warn({ url, screenId: screen.id }, "[crawler] Inline session-expired content detected \u2014 skipping DOM snapshot, marking requiresAuth");
           AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {
           });
@@ -2932,6 +2933,7 @@ async function crawlProject(opts, onProgress) {
                 const summaryPromise = (async () => {
                   try {
                     const llmCfg = await getCachedCrawlLLM(opts.tenantId);
+                    if (!llmCfg) return void 0;
                     const snippet = extracted.markdown.slice(0, 1200);
                     if (snippet.trim().length < 80) return void 0;
                     const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ""}:page-summary:${url}`);
@@ -2972,10 +2974,15 @@ ${snippet}`,
                   markdown: fitMarkdown ?? extracted.markdown,
                   heading: extracted.title || name
                 }).catch((err) => console.warn("[crawler] markdown extraction save failed:", err));
-                summaryPromise.then((aiSummary) => {
-                  if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => {
-                  });
-                });
+                pendingSummaryWrites.push(
+                  summaryPromise.then((aiSummary) => {
+                    if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch((e) => {
+                      logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, "[crawler] aiSummary DB save failed");
+                    });
+                  }).catch((e) => {
+                    logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, "[crawler] aiSummary generation failed");
+                  })
+                );
               }
               const structuredJson = summarizeStructuredElements(atElements);
               const artifact = buildCrawlArtifactEnvelope({
@@ -3413,7 +3420,7 @@ ${snippet}`,
     if (liteWorkerPromises.length > 0) await Promise.all(liteWorkerPromises);
     if (opts.discoveryMode) {
       return {
-        discoveredUrls: Array.from(visited.values()),
+        discoveredUrls: Array.from(visited.values()).filter((v) => v.startsWith("http")),
         screensFound: 0,
         screensErrored: 0,
         screensSkippedToNextSegment: 0,
@@ -3614,6 +3621,13 @@ ${snippet}`,
               await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
               });
             }
+          } else {
+            const p3aBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
+            if (INLINE_AUTH_WALL_RE.test(p3aBodyText)) {
+              logger.warn({ url: revisitUrl, screenId: existingScreenId }, "[P3a] Re-visit got 0 elements \u2014 inline session-expired content, marking requiresAuth");
+              AppBrain.updateScreenRequiresAuth(opts.tenantId, existingScreenId, true).catch(() => {
+              });
+            }
           }
           const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []);
           for (const href of revisitLinks) {
@@ -3753,6 +3767,14 @@ ${snippet}`,
                 await AppBrain.saveElement(opts.tenantId, opts.projectId, gapScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
                 });
               }
+              if (gapElems.length === 0) {
+                const gapBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
+                if (INLINE_AUTH_WALL_RE.test(gapBodyText)) {
+                  logger.warn({ url: gapUrl, screenId: gapScreen.id }, "[P6] Gap URL got 0 elements \u2014 inline session-expired content, marking requiresAuth");
+                  AppBrain.updateScreenRequiresAuth(opts.tenantId, gapScreen.id, true).catch(() => {
+                  });
+                }
+              }
               visited.set(canonicalize(gapUrl), gapScreen.id);
               discovered.push({ url: gapUrl, screenId: gapScreen.id, elements: gapElems.length, version: gapScreen.version });
               for (const lm of gapLinkMaps) {
@@ -4134,6 +4156,12 @@ ${snippet}`,
     }
     const screensSucceeded = discovered.filter((d) => !!d.screenId).length;
     const screensErrored = discovered.filter((d) => !!d.error && !d.screenId).length;
+    if (pendingSummaryWrites.length > 0) {
+      await Promise.race([
+        Promise.allSettled(pendingSummaryWrites),
+        new Promise((resolve) => setTimeout(resolve, Math.min(6e4, pendingSummaryWrites.length * 3e3)))
+      ]);
+    }
     if (screensSucceeded >= 3) {
       const screensWithElements = discovered.filter((d) => (d.elements ?? 0) > 0 && d.screenId).length;
       if (screensWithElements === 0) {
@@ -5867,6 +5895,16 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
     await AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, authDomPath, authDomCloudUrl).catch(() => {
     });
   }
+  const authDiscoveryElements = await readScreen(page).catch(() => []);
+  if (authDiscoveryElements.length > 0) {
+    await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+    });
+    for (const el of authDiscoveryElements) {
+      await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, void 0, void 0, true, void 0, void 0, void 0).catch(() => {
+      });
+    }
+    logger.info({ screenId: screen.id, elements: authDiscoveryElements.length }, "[crawler] auth-discovery: saved elements");
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,