@detiq/crawler 1.1.6 → 1.1.7

package/dist/crawler.js

@@ -10,7 +10,7 @@ import { z } from "zod";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { AppBrain, recordLLMCall } from "@detiq/app-brain";
-import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from "@detiq/agents";
+import { aiVisualCompare, askTracked, deterministicSeed, estimateCost, getTaskAgentLLMConfig } from "@detiq/agents";
 import { config } from "@detiq/core";
 import { pHashVerdict, mergeAiVisualVerdict } from "./visual-diff-verdict.js";
 import { skyvernFallback, skyvernNavigate } from "./skyvern.js";
@@ -795,6 +795,10 @@ async function crawlProject(opts, onProgress) {
     await page.waitForLoadState("networkidle", { timeout: 1e4 }).catch(() => {
     });
     await waitForStableDOM(page);
+    const homeFingerprintTitle = await page.title().catch(() => "");
+    const homeFingerprintElemCount = await page.evaluate(
+      () => document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
     if (!opts.discoveryMode) {
       try {
         await runAuthDiscoveryPhase(page, opts, new URL(opts.appUrl).origin, onProgress);
@@ -1275,6 +1279,93 @@ async function crawlProject(opts, onProgress) {
     let totalAttemptsThisSegment = 0;
     const maxAttempts = Math.max(segmentSize * 4, 200);
     let pendingSessionRestore = null;
+    const CRAWL_PAGE_PARALLELISM = Math.max(1, parseInt(process.env.CRAWL_PAGE_PARALLELISM ?? "5", 10));
+    async function runLiteWorker(workerPage) {
+      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        if (opts.abortSignal?.aborted) break;
+        if (++totalAttemptsThisSegment > maxAttempts) break;
+        const url = queue.shift();
+        if (!url) break;
+        const canonUrl = canonicalize(url);
+        if (visited.has(canonUrl)) continue;
+        try {
+          const { pathname } = new URL(url);
+          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+        } catch {
+          continue;
+        }
+        visited.set(canonUrl, "_pending_");
+        try {
+          const navErr = await workerPage.goto(url, { waitUntil: "domcontentloaded", timeout: 15e3 }).then(() => null, (e) => e);
+          if (navErr && /timeout/i.test(String(navErr?.message ?? navErr))) {
+            await workerPage.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(() => {
+            });
+          }
+          await workerPage.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+          });
+          if (homeFingerprintElemCount >= 0 && url !== startUrl) {
+            const wTitle = await workerPage.title().catch(() => "");
+            if (wTitle === homeFingerprintTitle && wTitle !== "") {
+              const wElemCount = await workerPage.evaluate(
+                () => document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+              ).catch(() => -1);
+              if (wElemCount >= 0 && wElemCount === homeFingerprintElemCount) {
+                console.log(`[lite-worker] skip ${url} \u2014 soft-404 (homepage fingerprint match)`);
+                visited.set(canonUrl, "_soft404_");
+                continue;
+              }
+            }
+          }
+          const pathParts = new URL(url).pathname.split("/").filter(Boolean);
+          const name = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
+          visited.set(canonUrl, screen.id);
+          const shotPath = path.join(opts.screenshotDir ?? "/tmp", `${screen.id}-desktop-v${screen.version}.png`);
+          const [elements] = await Promise.all([
+            domFallbackElements(workerPage).catch(() => []),
+            workerPage.screenshot({ path: shotPath, fullPage: true }).then(async () => {
+              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+              return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl);
+            }).catch(() => {
+            })
+          ]);
+          const nextLinks = await workerPage.evaluate((originStr) => {
+            return Array.from(document.querySelectorAll("a[href]")).map((a) => {
+              try {
+                return new URL(a.getAttribute("href"), window.location.href).href;
+              } catch {
+                return "";
+              }
+            }).filter((h) => h && new URL(h).origin === originStr && !h.includes("#")).slice(0, 200);
+          }, origin).catch(() => []);
+          for (const link of nextLinks) {
+            const cLink = canonicalize(link);
+            if (!visited.has(cLink)) {
+              queue.push(link);
+              edges.push({ fromUrl: url, toUrl: link });
+            }
+          }
+          discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+        } catch (err) {
+          logger.warn({ url, err: err?.message ?? String(err) }, "[lite-worker] URL processing failed");
+          discovered.push({ url, error: err?.message ?? String(err) });
+        }
+      }
+    }
+    const liteWorkerPromises = [];
+    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser) {
+      const pwCtx = playwrightBrowser.contexts()[0];
+      for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
+        const wp = await pwCtx.newPage();
+        await wp.addInitScript(() => {
+          Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+        }).catch(() => {
+        });
+        wp.setDefaultTimeout(45e3);
+        liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {
+        })));
+      }
+    }
     while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
       if (pendingSessionRestore) {
         await pendingSessionRestore();
@@ -1390,9 +1481,16 @@ async function crawlProject(opts, onProgress) {
               logger.info({ url }, "[crawler] temporarily cleared session for public auth route");
             }
           }
-          await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch((navErr) => {
-            logger.warn({ url, err: navErr?.message ?? String(navErr) }, "[crawler] navigation failed \u2014 skipping");
-          });
+          const navErr15 = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 15e3 }).then(() => null, (e) => e);
+          if (navErr15) {
+            if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
+              await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch((navErr) => {
+                logger.warn({ url, err: navErr?.message ?? String(navErr) }, "[crawler] navigation failed \u2014 skipping");
+              });
+            } else {
+              logger.warn({ url, err: navErr15?.message ?? String(navErr15) }, "[crawler] navigation failed \u2014 skipping");
+            }
+          }
           rawPage.off("response", onRedirectResponse);
           if (redirectHops.length > 0) {
             redirectRegistry.register({
@@ -1661,6 +1759,16 @@ async function crawlProject(opts, onProgress) {
           seenDomHashes.add(preDomHash);
           continue;
         }
+        if (homeFingerprintElemCount >= 0 && url !== startUrl && pageTitle === homeFingerprintTitle && pageTitle !== "") {
+          const pElemCount = await page.evaluate(
+            () => document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+          ).catch(() => -1);
+          if (pElemCount >= 0 && pElemCount === homeFingerprintElemCount) {
+            console.log(`[crawler] skip ${url} \u2014 soft-404 (homepage fingerprint match, title="${pageTitle}" elems=${pElemCount})`);
+            seenDomHashes.add(preDomHash);
+            continue;
+          }
+        }
         const rawPerf = await page.evaluate(() => {
           const nav = performance.getEntriesByType("navigation")[0];
           const paints = performance.getEntriesByType("paint");
@@ -1897,7 +2005,8 @@ async function crawlProject(opts, onProgress) {
           }),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
-          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0).then((freshLinks) => {
+          // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
+          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0 || preDomCount < 5).then((freshLinks) => {
             if (aiSkipCount > 0) aiSkipCount--;
             aiErrorStreak = 0;
             const merged = (cachedLinks ? Array.from(/* @__PURE__ */ new Set([...cachedLinks, ...freshLinks])) : freshLinks).filter((href) => isUsefulNavigationUrl(href, origin));
@@ -1987,6 +2096,7 @@ async function crawlProject(opts, onProgress) {
                   const llmCfg = await getCachedCrawlLLM(opts.tenantId);
                   const snippet = extracted.markdown.slice(0, 1200);
                   if (snippet.trim().length < 80) return void 0;
+                  const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ""}:page-summary:${url}`);
                   const { text, usage } = await askTracked(
                     "You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.",
                     `Screen: "${name}"
@@ -1995,7 +2105,9 @@ URL: ${url}
 Content:
 ${snippet}`,
                     llmCfg,
-                    200
+                    200,
+                    void 0,
+                    { temperature: 0, seed: summarySeed }
                   );
                   if (usage) {
                     recordLLMCall({
@@ -2394,6 +2506,7 @@ ${snippet}`,
         discovered.push({ url, error: err?.message ?? String(err) });
       }
     }
+    if (liteWorkerPromises.length > 0) await Promise.all(liteWorkerPromises);
     if (opts.discoveryMode) {
       return {
         discoveredUrls: Array.from(visited.values()),
@@ -3320,11 +3433,20 @@ async function contentFallbackElements(page) {
   return page.evaluate(() => {
     const out = [];
     const seen = /* @__PURE__ */ new Set();
-    const addUniq = (item) => {
+    const captureRect = (el) => {
+      try {
+        const r = el.getBoundingClientRect();
+        if (r.width === 0 && r.height === 0) return void 0;
+        return { x: r.x, y: r.y, width: r.width, height: r.height, top: r.top, right: r.right, bottom: r.bottom, left: r.left };
+      } catch {
+        return void 0;
+      }
+    };
+    const addUniq = (item, el) => {
       const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
       if (!seen.has(key) && item.meaning.trim().length > 1) {
         seen.add(key);
-        out.push(item);
+        out.push(el ? { ...item, boundingRect: captureRect(el) } : item);
       }
     };
     const isVisible = (el) => {
@@ -3342,7 +3464,7 @@ async function contentFallbackElements(page) {
     document.querySelectorAll("h1, h2, h3, h4, h5, h6").forEach((h) => {
       if (!isVisible(h)) return;
       const text = h.textContent?.replace(/\s+/g, " ").trim().slice(0, 100);
-      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
+      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) }, h);
     });
     document.querySelectorAll('details, [data-state="closed"], [data-state="open"]').forEach((el) => {
       if (!isVisible(el)) return;
@@ -3373,7 +3495,7 @@ async function contentFallbackElements(page) {
     document.querySelectorAll('[class*="card" i]:not(nav):not(header):not(footer), [class*="chip" i], [class*="badge" i], [class*="tag" i], [class*="pill" i]').forEach((el) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, " ").trim().slice(0, 150);
-      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: "content-card" });
+      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: "content-card" }, el);
     });
     const mainContent = document.querySelector('main, [role="main"], article') ?? document.body;
     let listItemCount = 0;
@@ -3382,14 +3504,14 @@ async function contentFallbackElements(page) {
       if (!isVisible(li)) return;
       const text = li.textContent?.replace(/\s+/g, " ").trim().slice(0, 150);
       if (text && text.length > 3 && text.length < 200) {
-        addUniq({ meaning: text, role: "list-item" });
+        addUniq({ meaning: text, role: "list-item" }, li);
         listItemCount++;
       }
     });
     mainContent.querySelectorAll('section > p:first-of-type, article > p:first-of-type, [class*="section" i] > p:first-of-type').forEach((p) => {
       if (!isVisible(p)) return;
       const text = p.textContent?.replace(/\s+/g, " ").trim().slice(0, 250);
-      if (text && text.length > 10) addUniq({ meaning: text, role: "section-paragraph" });
+      if (text && text.length > 10) addUniq({ meaning: text, role: "section-paragraph" }, p);
     });
     document.querySelectorAll('[class*="cta" i] p, [class*="call-to-action" i] p, section:not(form) > p').forEach((p) => {
       if (!isVisible(p)) return;
@@ -3774,7 +3896,7 @@ async function contentFallbackElements(page) {
     document.querySelectorAll('[role="alert"], [role="status"], [role="banner"]:not(header), .alert, .notice, .notification, .warning-banner, .error-banner, .success-banner, [class*="alert-" i]:not(button), [class*="toast" i]:not(button), [class*="snackbar" i]').forEach((el) => {
       if (!isVisible(el)) return;
       const text = el.textContent?.replace(/\s+/g, " ").trim().slice(0, 150);
-      if (text && text.length > 5) addUniq({ meaning: text, role: "alert-banner", notes: JSON.stringify({ ariaRole: el.getAttribute("role") }) });
+      if (text && text.length > 5) addUniq({ meaning: text, role: "alert-banner", notes: JSON.stringify({ ariaRole: el.getAttribute("role") }) }, el);
     });
     let pricingCount = 0;
     document.querySelectorAll('[class*="pricing" i], [class*="plan" i]:not(main):not(article), [class*="tier" i]:not(nav), [class*="package" i]:not(script)').forEach((el) => {
@@ -4334,7 +4456,7 @@ async function hasMeaningfulAuthOrPublicState(page, requestedUrl) {
       return false;
     }
   })();
-  const domLooksRelevant = await page.evaluate(() => {
+  const domState = await page.evaluate(() => {
     const text = (document.body?.innerText ?? "").toLowerCase();
     const inputs = Array.from(document.querySelectorAll("input"));
     const hasPassword = inputs.some((i) => i.type === "password");
@@ -4344,8 +4466,11 @@ async function hasMeaningfulAuthOrPublicState(page, requestedUrl) {
     });
     const hasAuthCopy = /(sign in|log in|login|sign up|create account|register|forgot password|reset password|verify email|one-time|otp|continue with email)/i.test(text);
     const hasPublicNav = /(pricing|contact|features|documentation|docs|about|privacy|terms)/i.test(text);
-    return hasPassword || hasIdentityInput || hasAuthCopy || hasPublicNav;
-  }).catch(() => false);
+    const isSoft404 = !hasPassword && !hasIdentityInput && /(page not found|404|this page (could not|doesn'?t) (be found|exist)|we couldn'?t find|no longer exists|been moved|doesn'?t exist)/i.test(text);
+    return { hasPassword, hasIdentityInput, hasAuthCopy, hasPublicNav, isSoft404 };
+  }).catch(() => ({ hasPassword: false, hasIdentityInput: false, hasAuthCopy: false, hasPublicNav: false, isSoft404: false }));
+  if (domState.isSoft404) return false;
+  const domLooksRelevant = domState.hasPassword || domState.hasIdentityInput || domState.hasAuthCopy || domState.hasPublicNav;
   return pathLooksRelevant || domLooksRelevant;
 }
 async function discoverAuthCandidates(page, origin) {
@@ -4490,6 +4615,18 @@ async function runAuthDiscoveryPhase(page, opts, origin, onProgress) {
     } catch {
     }
   }
+  let homeTitle = "";
+  let homeElemCount = -1;
+  try {
+    await page.goto(opts.appUrl, { waitUntil: "domcontentloaded", timeout: 15e3 }).catch(() => null);
+    await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
+    });
+    homeTitle = await page.title().catch(() => "");
+    homeElemCount = await page.evaluate(
+      () => document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+    ).catch(() => -1);
+  } catch {
+  }
   let captured = 0;
   let ordinal = 0;
   await onProgress?.(10, "Discovering public and auth flows");
@@ -4503,6 +4640,13 @@ async function runAuthDiscoveryPhase(page, opts, origin, onProgress) {
       await waitForStableDOM(page);
       await acceptCookieConsent(page, { waitMs: 400 }).catch(() => {
       });
+      if (current.source === "seed_path" && homeElemCount >= 0) {
+        const pageTitle = await page.title().catch(() => "");
+        const elemCount = await page.evaluate(
+          () => document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
+        ).catch(() => -1);
+        if (pageTitle === homeTitle && elemCount === homeElemCount) continue;
+      }
       const finalUrl = page.url?.() ?? current.url;
       const captureKey = normalizeAuthUrl(finalUrl);
       if (!capturedKeys.has(captureKey)) {