@detiq/crawler 1.1.21 → 1.1.23

package/Dockerfile

@@ -36,6 +36,11 @@ COPY packages/vscode-extension/package.json packages/vscode-extension/
 COPY packages/web/package.json          packages/web/
 RUN npm ci --include=dev
 
+# Patch Stagehand's act() schema to add `additionalProperties: false` on the
+# inner anyOf[0] object. OpenAI strict mode (v1/responses) requires it on every
+# nested object; Stagehand ≤3.6 omits it, causing a 400 "invalid_json_schema".
+RUN node -e "const fs=require('fs');['packages/crawler/node_modules/@browserbasehq/stagehand/dist/cjs/lib/inference.js','packages/crawler/node_modules/@browserbasehq/stagehand/dist/esm/lib/inference.js'].forEach(f=>{if(!fs.existsSync(f)){console.log('skip',f);return;}let c=fs.readFileSync(f,'utf8');if(c.includes('.strict()')){console.log('already patched',f);return;}c=c.replace(/\\.nullable\\(\\)\\n(\\s+)\\.describe\\(\"The element to act on/g,'\.strict()\n\$1.nullable()\n\$1.describe(\"The element to act on');fs.writeFileSync(f,c);console.log('patched',f);});"
+
 # Re-install chromium after npm ci so the browser revision matches the
 # exact playwright version resolved by the lockfile (not the base image's
 # bundled version, which may differ if the lockfile was updated).

package/dist/crawler.js

@@ -73,7 +73,7 @@ const _crawlLLMCache = /* @__PURE__ */ new Map();
 async function getCachedCrawlLLM(tenantId) {
   if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
   const cfg = await getTaskAgentLLMConfig(tenantId, "crawl").catch(() => null);
-  _crawlLLMCache.set(tenantId, cfg);
+  if (cfg) _crawlLLMCache.set(tenantId, cfg);
   return cfg;
 }
 function getGitInfo() {
@@ -88,7 +88,7 @@ function getGitInfo() {
 const ALLOWED_NUMERIC_PATHS = /* @__PURE__ */ new Set(["/2fa", "/404", "/500", "/403", "/401"]);
 const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
-const NUMERIC_ID_PATH_SEGMENT_RE = /^\d[\d-]*$/;
+const NUMERIC_ID_PATH_SEGMENT_RE = /^\d{4,}$/;
 const COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE = /^\d+-\d+$/;
 function normalizePathForNumericGuard(pathname) {
   const normalized = pathname.replace(/\/+$/, "") || "/";
@@ -1598,6 +1598,7 @@ async function crawlProject(opts, onProgress) {
     };
     const edges = [];
     const pendingTransitions = [];
+    const pendingSummaryWrites = [];
     const discovered = [];
     const performanceMetrics = [];
     const seenDomHashes = new Set(opts.knownDomHashes ?? []);
@@ -2932,6 +2933,7 @@ async function crawlProject(opts, onProgress) {
                 const summaryPromise = (async () => {
                   try {
                     const llmCfg = await getCachedCrawlLLM(opts.tenantId);
+                    if (!llmCfg) return void 0;
                     const snippet = extracted.markdown.slice(0, 1200);
                     if (snippet.trim().length < 80) return void 0;
                     const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ""}:page-summary:${url}`);
@@ -2972,10 +2974,15 @@ ${snippet}`,
                   markdown: fitMarkdown ?? extracted.markdown,
                   heading: extracted.title || name
                 }).catch((err) => console.warn("[crawler] markdown extraction save failed:", err));
-                summaryPromise.then((aiSummary) => {
-                  if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch(() => {
-                  });
-                });
+                pendingSummaryWrites.push(
+                  summaryPromise.then((aiSummary) => {
+                    if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch((e) => {
+                      logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, "[crawler] aiSummary DB save failed");
+                    });
+                  }).catch((e) => {
+                    logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, "[crawler] aiSummary generation failed");
+                  })
+                );
               }
               const structuredJson = summarizeStructuredElements(atElements);
               const artifact = buildCrawlArtifactEnvelope({
@@ -3413,7 +3420,7 @@ ${snippet}`,
     if (liteWorkerPromises.length > 0) await Promise.all(liteWorkerPromises);
     if (opts.discoveryMode) {
       return {
-        discoveredUrls: Array.from(visited.values()),
+        discoveredUrls: Array.from(visited.values()).filter((v) => v.startsWith("http")),
         screensFound: 0,
         screensErrored: 0,
         screensSkippedToNextSegment: 0,
@@ -4149,6 +4156,12 @@ ${snippet}`,
     }
     const screensSucceeded = discovered.filter((d) => !!d.screenId).length;
     const screensErrored = discovered.filter((d) => !!d.error && !d.screenId).length;
+    if (pendingSummaryWrites.length > 0) {
+      await Promise.race([
+        Promise.allSettled(pendingSummaryWrites),
+        new Promise((resolve) => setTimeout(resolve, Math.min(6e4, pendingSummaryWrites.length * 3e3)))
+      ]);
+    }
     if (screensSucceeded >= 3) {
       const screensWithElements = discovered.filter((d) => (d.elements ?? 0) > 0 && d.screenId).length;
       if (screensWithElements === 0) {