@detiq/crawler 1.1.23 → 1.1.25

package/dist/crawler.js

@@ -54,7 +54,7 @@ import { enumerateDropdowns } from "./dropdown-enumerator.js";
 import { probeCrudFlows } from "./crud-interaction.js";
 import { crawlFiltersAndPagination } from "./filter-pagination-crawler.js";
 import { probeAdvancedUI } from "./advanced-ui-probes.js";
-import { ScreenTransitionGraph } from "./state-graph.js";
+import { ScreenTransitionGraph, simHash32 } from "./state-graph.js";
 import { crawlNestedContexts } from "./nested-window-crawler.js";
 import { crawlShadowDom } from "./shadow-dom-crawler.js";
 import { crawlAccessibilityTree } from "./accessibility-tree-crawler.js";
@@ -86,6 +86,7 @@ function getGitInfo() {
   }
 }
 const ALLOWED_NUMERIC_PATHS = /* @__PURE__ */ new Set(["/2fa", "/404", "/500", "/403", "/401"]);
+let _activeSeedPatterns = [];
 const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
 const NUMERIC_ID_PATH_SEGMENT_RE = /^\d{4,}$/;
@@ -110,7 +111,7 @@ function isUsefulNavigationUrl(href, siteOrigin) {
     const u = new URL(href);
     if (!sameSite(u.toString(), siteOrigin)) return false;
     if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
-    if (isGeneratedDynamicPath(u.pathname)) return false;
+    if (isGeneratedDynamicPath(u.pathname) && !_activeSeedPatterns.some((re) => re.test(u.pathname))) return false;
     if (DOWNLOAD_EXT_RE.test(u.pathname)) return false;
     if (API_PATH_RE.test(u.pathname)) return false;
     if (EXPORT_PATH_RE.test(u.pathname)) return false;
@@ -119,6 +120,13 @@ function isUsefulNavigationUrl(href, siteOrigin) {
     return false;
   }
 }
+function buildSeedPatternRegexes(patterns) {
+  return patterns.map((p) => {
+    const parts = p.split("{id}").map((part) => part.replace(/[.+^${}()|[\]\\]/g, "\\$&"));
+    const body = parts.join("[^/]+");
+    return new RegExp(`^${body}(/.*)?$`, "i");
+  });
+}
 const AUTH_DISCOVERY_MAX_SCREENS = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_SCREENS ?? "") || 36;
 const AUTH_DISCOVERY_MAX_QUEUE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_QUEUE ?? "") || 48;
 const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE ?? "") || 6;
@@ -244,9 +252,10 @@ async function capturePageHtml(page) {
   }
   return html;
 }
-async function launchChromeCDP(execPath, extraArgs, proxy) {
+async function launchChromeCDP(execPath, extraArgs, proxy, useStealthPlugin = true) {
   const port = 9200 + Math.floor(Math.random() * 700);
-  const browserServer = await stealthChromium.launchServer({
+  const launcher = useStealthPlugin ? stealthChromium : chromium;
+  const browserServer = await launcher.launchServer({
     headless: true,
     executablePath: execPath,
     // Use Playwright's native proxy option — it handles 407 auth challenges automatically.
@@ -377,6 +386,7 @@ const TwoFASchema = z.object({
 const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
 async function crawlProject(opts, onProgress) {
   const tuning = await getCrawlTuning();
+  _activeSeedPatterns = buildSeedPatternRegexes(opts.seedPathPatterns ?? []);
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
   const uploadToCloud = async (localPath, key) => {
     if (!opts.storageConfig) return void 0;
@@ -643,7 +653,7 @@ async function crawlProject(opts, onProgress) {
       logger.warn({ err: foxErr }, "[crawler] Camoufox bootstrap failed \u2014 continuing without it");
     }
   }
-  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy);
+  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy, opts.stealth !== false);
   let stagehandPromptTokens = 0;
   let stagehandCompletionTokens = 0;
   const stagehandLogger = (line) => {
@@ -1444,7 +1454,7 @@ async function crawlProject(opts, onProgress) {
       ...sitemapUrls.filter((u) => canonicalize(u) !== canonicalize(startUrl))
     ];
     const queued = new Set(seedUrls.map(canonicalize));
-    const queue = new CrawlFrontier(opts.traversalStrategy ?? "BFS", seedUrls, opts.bestFirstKeywords, opts.urlFilter);
+    const queue = new CrawlFrontier(opts.traversalStrategy ?? "BFS", seedUrls, opts.bestFirstKeywords, opts.urlFilter, opts.includePatterns, opts.excludePatterns);
     if (selective) {
       queue.push = () => {
       };
@@ -1467,7 +1477,7 @@ async function crawlProject(opts, onProgress) {
             const { pathname } = new URL(u);
             const stripWww = (h) => h.replace(/^www\./i, "").toLowerCase();
             if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
-            if (isGeneratedDynamicPath(pathname)) return false;
+            if (isGeneratedDynamicPath(pathname) && !_activeSeedPatterns.some((re) => re.test(pathname))) return false;
             if (!isUsefulNavigationUrl(u, startUrl)) return false;
             const canon = canonicalize(u);
             if (seenReseedCanon.has(canon)) return false;
@@ -1629,7 +1639,7 @@ async function crawlProject(opts, onProgress) {
         if (visited.has(canonUrl)) continue;
         try {
           const { pathname } = new URL(url);
-          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname) && !_activeSeedPatterns.some((re) => re.test(pathname))) continue;
           if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
         } catch {
           continue;
@@ -1985,7 +1995,7 @@ async function crawlProject(opts, onProgress) {
           console.log(`[crawler] skip content page: ${url}`);
           continue;
         }
-        if (isGeneratedDynamicPath(pathname)) {
+        if (isGeneratedDynamicPath(pathname) && !_activeSeedPatterns.some((re) => re.test(pathname))) {
           console.log(`[crawler] skip generated dynamic route: ${url}`);
           continue;
         }
@@ -2724,10 +2734,24 @@ async function crawlProject(opts, onProgress) {
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
           // Note: preDomCount < aiMinElements guard removed — it was backwards: sparse pages (SPAs) need AI most.
-          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).then((freshLinks) => {
+          discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).then((metas) => {
+            if (metas.length > 0) {
+              AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, screen.id, metas.map((m) => ({
+                href: m.url,
+                text: m.text,
+                landmark: m.landmark,
+                linkType: "NAV_LINK",
+                source: m.source,
+                jobId: opts.jobId
+              }))).catch(() => {
+              });
+            }
+            return metas.map((m) => m.url);
+          }).then((freshLinks) => {
             if (aiSkipCount > 0) aiSkipCount--;
             aiErrorStreak = 0;
-            const merged = (cachedLinks ? Array.from(/* @__PURE__ */ new Set([...cachedLinks, ...freshLinks])) : freshLinks).filter((href) => isUsefulNavigationUrl(href, origin));
+            const freshUrls = freshLinks;
+            const merged = (cachedLinks ? Array.from(/* @__PURE__ */ new Set([...cachedLinks, ...freshUrls])) : freshUrls).filter((href) => isUsefulNavigationUrl(href, origin));
             logger.info({ url, domLinks: freshLinks.length, cached: cachedLinks?.length ?? 0, merged: merged.length }, "[crawler] link-discovery: result");
             if (merged.length > 0) {
               AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
@@ -2741,7 +2765,7 @@ async function crawlProject(opts, onProgress) {
             }
             return merged;
           }).catch((err) => {
-            const fallback = err?.domLinks ?? [];
+            const fallback = (err?.domLinks ?? []).map((l) => l.url);
             if (err?.isRateLimit) {
               aiErrorStreak++;
               if (aiErrorStreak >= 5) {
@@ -3269,7 +3293,7 @@ ${snippet}`,
               if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 3) {
                 try {
                   const hrefPath = new URL(href).pathname;
-                  if (isGeneratedDynamicPath(hrefPath)) continue;
+                  if (isGeneratedDynamicPath(hrefPath) && !_activeSeedPatterns.some((re) => re.test(hrefPath))) continue;
                 } catch {
                 }
                 if (robotsCache && !await robotsCache.isAllowed(href)) continue;
@@ -3473,7 +3497,20 @@ ${snippet}`,
             }
             const [elems, links, linkMappings] = await Promise.all([
               extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0),
-              discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0),
+              discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).then((metas) => {
+                if (metas.length > 0) {
+                  AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, screen.id, metas.map((m) => ({
+                    href: m.url,
+                    text: m.text,
+                    landmark: m.landmark,
+                    linkType: "NAV_LINK",
+                    source: m.source,
+                    jobId: opts.jobId
+                  }))).catch(() => {
+                  });
+                }
+                return metas.map((m) => m.url);
+              }),
               extractLinkMappings(page, origin)
             ]);
             for (const el of elems) {
@@ -3485,7 +3522,8 @@ ${snippet}`,
             for (const lm of linkMappings) {
               pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
             }
-            for (const href of links) {
+            for (const link of links) {
+              const href = link;
               const c = canonicalize(href);
               if (isUsefulNavigationUrl(href, origin) && !queued.has(c) && !visited.has(c)) {
                 queued.add(c);
@@ -3629,7 +3667,20 @@ ${snippet}`,
               });
             }
           }
-          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []);
+          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).then((metas) => {
+            if (metas.length > 0) {
+              AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, existingScreenId, metas.map((m) => ({
+                href: m.url,
+                text: m.text,
+                landmark: m.landmark,
+                linkType: "NAV_LINK",
+                source: m.source,
+                jobId: opts.jobId
+              }))).catch(() => {
+              });
+            }
+            return metas.map((m) => m.url);
+          }).catch(() => []);
           for (const href of revisitLinks) {
             const canon = canonicalize(href);
             if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
@@ -3680,7 +3731,20 @@ ${snippet}`,
           }
           const [p3bElems, p3bLinks] = await Promise.all([
             extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []),
-            discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).catch(() => [])
+            discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0, rawPage ?? void 0).then((metas) => {
+              if (metas.length > 0) {
+                AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, screen.id, metas.map((m) => ({
+                  href: m.url,
+                  text: m.text,
+                  landmark: m.landmark,
+                  linkType: "NAV_LINK",
+                  source: m.source,
+                  jobId: opts.jobId
+                }))).catch(() => {
+                });
+              }
+              return metas.map((m) => m.url);
+            }).catch(() => [])
           ]);
           if (p3bElems.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
@@ -3760,7 +3824,20 @@ ${snippet}`,
               }
               const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
                 extractElements(stagehand, page, elementLLM ?? void 0, rawPage ?? void 0).catch(() => []),
-                discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []),
+                discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).then((metas) => {
+                  if (metas.length > 0) {
+                    AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, gapScreen.id, metas.map((m) => ({
+                      href: m.url,
+                      text: m.text,
+                      landmark: m.landmark,
+                      linkType: "NAV_LINK",
+                      source: m.source,
+                      jobId: opts.jobId
+                    }))).catch(() => {
+                    });
+                  }
+                  return metas.map((m) => m.url);
+                }).catch(() => []),
                 extractLinkMappings(page, origin).catch(() => [])
               ]);
               for (const el of gapElems) {
@@ -3882,6 +3959,23 @@ ${snippet}`,
       }
     }).length;
     logger.info({ transitionCount }, "KAG: wired element\u2192screen transitions");
+    if (opts.jobId) {
+      const hrefToScreenId = /* @__PURE__ */ new Map();
+      for (const [canonUrl, screenId] of visited) {
+        if (screenId && screenId !== "_pending_" && screenId !== "_error_") {
+          hrefToScreenId.set(canonUrl, screenId);
+          try {
+            const u = new URL(canonUrl);
+            hrefToScreenId.set(u.href, screenId);
+            hrefToScreenId.set(u.origin + u.pathname, screenId);
+            hrefToScreenId.set(u.origin + u.pathname.replace(/\/$/, ""), screenId);
+          } catch {
+          }
+        }
+      }
+      AppBrain.resolveDiscoveredLinkTargets(opts.tenantId, opts.projectId, opts.jobId, hrefToScreenId).catch(() => {
+      });
+    }
     AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
       jobId: opts.jobId,
       type: "WORKFLOW_GRAPH_SUMMARY",
@@ -4052,11 +4146,33 @@ ${snippet}`,
     if (shouldRunInteractiveProbe) {
       await onProgress?.(88, "Running interactive probe (capturing real errors & modals)");
       const probeInputs = discovered.filter((d) => d.screenId).map((d) => ({ url: d.url, screenId: d.screenId, elements: d.elementFacts ?? [] }));
-      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
+      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress, opts.jobId);
     }
     try {
       await onProgress?.(90, "Generating state graph coverage report");
       const stateGraph = new ScreenTransitionGraph();
+      const now = (/* @__PURE__ */ new Date()).toISOString();
+      for (const d of discovered) {
+        if (!d.screenId || !d.url) continue;
+        const hashVal = simHash32(d.url);
+        stateGraph.addState({
+          id: d.screenId,
+          url: d.url,
+          normalizedUrl: d.url,
+          title: d.url,
+          interactiveElementCount: d.elements ?? 0,
+          domFingerprint: hashVal.toString(16).padStart(8, "0"),
+          simHashValue: hashVal,
+          visitedAt: now
+        });
+      }
+      for (const edge of edges) {
+        const fromId = visited.get(canonicalize(edge.fromUrl));
+        const toId = visited.get(canonicalize(edge.toUrl));
+        if (fromId && toId && fromId !== "_pending_" && fromId !== "_error_" && toId !== "_pending_" && toId !== "_error_") {
+          stateGraph.addTransition(fromId, toId, "navigate", edge.toUrl, now);
+        }
+      }
       const coveragePayload = {
         stateGraph: stateGraph.getCoverageReport(),
         totalScreensCrawled: discovered.filter((d) => !!d.screenId).length
@@ -4313,7 +4429,8 @@ async function extractElements(stagehand, page, llmCfg, rawPage) {
 async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false, rawPage) {
   const navExpanded = await expandNavMenus(page);
   const domLinks = await domFallbackLinks(page, origin, rawPage);
-  const interactiveLinks = await interactAndCollectLinks(page, origin);
+  const interactiveRaw = await interactAndCollectLinks(page, origin);
+  const interactiveLinks = interactiveRaw.map((url) => ({ url, source: "INTERACTIVE" }));
   if (navExpanded) {
     page.keyboard?.press("Escape").catch(() => {
     });
@@ -4327,7 +4444,14 @@ async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false, ra
     }).catch(() => {
     });
   }
-  const allDomLinks = Array.from(/* @__PURE__ */ new Set([...domLinks, ...interactiveLinks]));
+  const seenUrls = /* @__PURE__ */ new Set();
+  const allDomLinks = [];
+  for (const link of [...domLinks, ...interactiveLinks]) {
+    if (!seenUrls.has(link.url)) {
+      seenUrls.add(link.url);
+      allDomLinks.push(link);
+    }
+  }
   if (allDomLinks.length >= 50 || skipAi) return allDomLinks;
   try {
     const extracted = await Promise.race([
@@ -4351,8 +4475,16 @@ async function discoverLinks(stagehand, origin, page, llmCfg, skipAi = false, ra
     }).filter((u) => {
       if (!u) return false;
       return isUsefulNavigationUrl(u, origin);
-    });
-    return Array.from(/* @__PURE__ */ new Set([...allDomLinks, ...aiLinks])).filter((href) => isUsefulNavigationUrl(href, origin));
+    }).map((url) => ({ url, source: "AI" }));
+    const merged = [...allDomLinks];
+    const mergedUrls = new Set(merged.map((l) => l.url));
+    for (const link of aiLinks) {
+      if (!mergedUrls.has(link.url) && isUsefulNavigationUrl(link.url, origin)) {
+        mergedUrls.add(link.url);
+        merged.push(link);
+      }
+    }
+    return merged.filter((l) => isUsefulNavigationUrl(l.url, origin));
   } catch (err) {
     const is429 = err?.status === 429 || String(err?.message ?? "").includes("429") || String(err?.message ?? "").includes("rate limit");
     if (is429) throw Object.assign(new Error("RATE_LIMITED"), { isRateLimit: true, domLinks: allDomLinks });
@@ -5442,7 +5574,7 @@ async function domFallbackLinks(page, origin, rawPage) {
   function _linkEval(originStr) {
     const logout = /logout|log out|sign out|delete/i;
     const isRouteHash = (hash) => /^#!?\//.test(hash);
-    const seen = /* @__PURE__ */ new Set();
+    const seen = /* @__PURE__ */ new Map();
     const add = (rawHref, textContent) => {
       if (logout.test(textContent)) return;
       if (!rawHref) return;
@@ -5451,7 +5583,7 @@ async function domFallbackLinks(page, origin, rawPage) {
       try {
         const u = new URL(rawHref, location.href);
         if (!isRouteHash(u.hash)) u.hash = "";
-        if (new URL(u).origin === originStr) seen.add(u.toString());
+        if (new URL(u).origin === originStr) seen.set(u.toString(), (textContent || "").trim().slice(0, 120));
       } catch {
       }
     };
@@ -5485,7 +5617,20 @@ async function domFallbackLinks(page, origin, rawPage) {
       } catch (_) {
       }
     }
-    return Array.from(seen);
+    return Array.from(seen.entries()).map(([url, text]) => ({ url, text }));
+  }
+  function inferLandmark(url) {
+    const p = url.toLowerCase();
+    if (p.includes("/auth") || p.includes("/login") || p.includes("/register")) return "nav";
+    return void 0;
+  }
+  function toMeta(raw) {
+    return raw.map(({ url, text }) => ({
+      url,
+      text: text || void 0,
+      landmark: inferLandmark(url),
+      source: "DOM"
+    }));
   }
   const _linkEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_linkEval.toString()})(${JSON.stringify(origin)})})();`;
   if (rawPage && typeof rawPage.context === "function") {
@@ -5500,7 +5645,7 @@ async function domFallbackLinks(page, origin, rawPage) {
             awaitPromise: false
           });
           if (!evalResult?.exceptionDetails && Array.isArray(evalResult?.result?.value)) {
-            return evalResult.result.value;
+            return toMeta(evalResult.result.value);
           }
           if (evalResult?.exceptionDetails) {
             console.warn("[crawler] domFallbackLinks CDP error:", evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
@@ -5515,7 +5660,7 @@ async function domFallbackLinks(page, origin, rawPage) {
     }
   }
   const evalPage = rawPage ?? page;
-  return evalPage.evaluate(_linkEvalStr).catch((err) => {
+  return evalPage.evaluate(_linkEvalStr).then(toMeta).catch((err) => {
     console.warn("[crawler] domFallbackLinks page.evaluate threw:", err?.message ?? String(err));
     return [];
   });