@detiq/crawler 1.1.22 → 1.1.24

package/src/crawler.ts

@@ -79,7 +79,7 @@ import { probeCrudFlows } from './crud-interaction.js';
 import { crawlFiltersAndPagination } from './filter-pagination-crawler.js';
 import { probeMultiContext } from './multi-context-handler.js';
 import { probeAdvancedUI } from './advanced-ui-probes.js';
-import { ScreenTransitionGraph } from './state-graph.js';
+import { ScreenTransitionGraph, simHash32 } from './state-graph.js';
 import { crawlNestedContexts } from './nested-window-crawler.js';
 import { crawlShadowDom } from './shadow-dom-crawler.js';
 import { crawlAccessibilityTree } from './accessibility-tree-crawler.js';
@@ -125,7 +125,18 @@ interface PerformanceMetric {
   cls?: number;
 }
 
+export interface DiscoveredLinkMeta {
+  url: string;
+  text?: string;
+  landmark?: string;
+  source: 'DOM' | 'AI' | 'INTERACTIVE';
+}
+
 const ALLOWED_NUMERIC_PATHS = new Set(['/2fa', '/404', '/500', '/403', '/401']);
+
+// Seed path patterns compiled for the active crawl. Set by crawlProject() at the
+// start of each run. All isUsefulNavigationUrl() calls share this for the duration.
+let _activeSeedPatterns: RegExp[] = [];
 const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
 // Match 4+ digit pure numeric IDs only — aligns with canonicalize()'s /\d{4,}/ threshold.
@@ -171,7 +182,9 @@ function isUsefulNavigationUrl(href: string, siteOrigin: string): boolean {
     const u = new URL(href);
     if (!sameSite(u.toString(), siteOrigin)) return false;
     if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
-    if (isGeneratedDynamicPath(u.pathname)) return false;
+    // Bypass CUID/UUID guard for seed path patterns declared by the caller.
+    // _activeSeedPatterns is set per-crawl by crawlProject.
+    if (isGeneratedDynamicPath(u.pathname) && !_activeSeedPatterns.some(re => re.test(u.pathname))) return false;
     if (DOWNLOAD_EXT_RE.test(u.pathname)) return false;
     if (API_PATH_RE.test(u.pathname)) return false;
     if (EXPORT_PATH_RE.test(u.pathname)) return false;
@@ -181,6 +194,19 @@ function isUsefulNavigationUrl(href: string, siteOrigin: string): boolean {
   }
 }
 
+/**
+ * Compile seedPathPatterns (e.g., ['/projects/{id}/tests']) into RegExp matchers.
+ * {id} matches any single path segment (CUID, UUID, numeric, slug, etc.).
+ * Example: '/projects/{id}/tests' → /^\/projects\/[^/]+\/tests(\/.*)?$/i
+ */
+function buildSeedPatternRegexes(patterns: string[]): RegExp[] {
+  return patterns.map((p) => {
+    const parts = p.split('{id}').map(part => part.replace(/[.+^${}()|[\]\\]/g, '\\$&'));
+    const body = parts.join('[^/]+');
+    return new RegExp(`^${body}(/.*)?$`, 'i');
+  });
+}
+
 type AuthFlowType = 'public' | 'login' | 'register' | 'password_reset' | 'verification' | 'invite' | 'session_recovery';
 type AuthDiscoverySource = 'entry' | 'seed_path' | 'page_link' | 'auth_click';
 type AuthDiscoveryProgress = (pct: number, step: string) => void | Promise<void>;
@@ -327,11 +353,15 @@ async function launchChromeCDP(
   execPath: string,
   extraArgs: string[],
   proxy?: { server: string; username?: string; password?: string },
+  useStealthPlugin = true,
 ): Promise<{ wsUrl: string; proc: any }> {
   // Use a random port in 9200-9900 range to avoid clashes between concurrent crawls.
   const port = 9200 + Math.floor(Math.random() * 700);
 
-  const browserServer = await stealthChromium.launchServer({
+  // Use stealthChromium (playwright-extra + puppeteer-extra-plugin-stealth) by default.
+  // Pass useStealthPlugin=false to opt out for a specific crawl job (opts.stealth === false).
+  const launcher = useStealthPlugin ? stealthChromium : chromium;
+  const browserServer = await launcher.launchServer({
     headless: true,
     executablePath: execPath,
     // Use Playwright's native proxy option — it handles 407 auth challenges automatically.
@@ -503,6 +533,13 @@ export async function crawlProject(opts: {
   startUrls?: string[];
   /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
   selectiveUrls?: string[];
+  /**
+   * URL path patterns that bypass the CUID/UUID generated-ID filter during BFS link discovery.
+   * Use {id} as a placeholder for any generated path segment (CUID, UUID, numeric ID, etc.).
+   * Example: ['/projects/{id}/tests', '/workspaces/{id}/settings']
+   * Without this, all URLs containing generated IDs are silently skipped by the BFS.
+   */
+  seedPathPatterns?: string[];
   /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
   interactive?: boolean;
   /** Optional abort signal — abort() cancels the crawl between page navigations. */
@@ -586,10 +623,17 @@ export async function crawlProject(opts: {
   autoThrottle?: boolean;
   /** Regex pattern — only crawl URLs whose path+query match this pattern. */
   urlFilter?: string;
+  /** Glob patterns — only enqueue URLs whose pathname matches at least one (supports * and **). */
+  includePatterns?: string[];
+  /** Glob patterns — skip URLs whose pathname matches any of these (supports * and **). */
+  excludePatterns?: string[];
   /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
   skipUrls?: string[];
   /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
   knownDomHashes?: string[];
+  /** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
+   *  Default true — stealth is applied unless explicitly set to false. */
+  stealth?: boolean;
   /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
   onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
   /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
@@ -597,6 +641,10 @@ export async function crawlProject(opts: {
 }, onProgress?: (pct: number, step: string) => Promise<void>) {
   const tuning = await getCrawlTuning();
 
+  // Compile seed path patterns for this crawl run. Applied by isUsefulNavigationUrl()
+  // to bypass the CUID guard for explicitly declared URL shapes.
+  _activeSeedPatterns = buildSeedPatternRegexes(opts.seedPathPatterns ?? []);
+
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
 
   // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
@@ -908,7 +956,7 @@ export async function crawlProject(opts: {
     }
   }
 
-  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy);
+  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy, opts.stealth !== false);
 
   // Accumulate token usage from Stagehand's internal LLM calls (extract/act/observe).
   // Stagehand doesn't expose per-call usage externally; we parse it from its logger.
@@ -1354,6 +1402,7 @@ export async function crawlProject(opts: {
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, finalUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
           if (!screen) continue;
           const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
+          // TODO GAP-09: pass opts.screenshotConfig (width/height/quality) here when screenshotConfig is wired into the opts type.
           await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {});
           const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
           await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl).catch(() => {});
@@ -1787,7 +1836,7 @@ export async function crawlProject(opts: {
           ...sitemapUrls.filter((u) => canonicalize(u) !== canonicalize(startUrl)),
         ];
     const queued = new Set<string>(seedUrls.map(canonicalize));
-    const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
+    const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter, opts.includePatterns, opts.excludePatterns);
     // Selective recrawl: block all link-discovery pushes so BFS only visits the requested URLs.
     // Constructor seeding already ran above — this no-op override fires only for BFS-loop discoveries.
     if (selective) {
@@ -1824,7 +1873,8 @@ export async function crawlProject(opts: {
               const stripWww = (h: string) => h.replace(/^www\./i, '').toLowerCase();
               if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
               // Don't re-seed generated dynamic routes — they produce duplicate templates.
-              if (isGeneratedDynamicPath(pathname)) return false;
+              // Exception: seed path patterns explicitly declared by the caller.
+              if (isGeneratedDynamicPath(pathname) && !_activeSeedPatterns.some(re => re.test(pathname))) return false;
               // Don't re-seed API/export/download paths — they were misidentified as screens.
               if (!isUsefulNavigationUrl(u, startUrl)) return false;
               // Canonical dedup: mark seen immediately during filter so duplicates are caught.
@@ -2018,7 +2068,7 @@ export async function crawlProject(opts: {
 
         try {
           const { pathname } = new URL(url);
-          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+          if (SKIP_PATH.test(pathname) || (isGeneratedDynamicPath(pathname) && !_activeSeedPatterns.some(re => re.test(pathname)))) continue;
           // Skip API/export paths — these return data files, not navigable pages
           if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
         } catch { continue; }
@@ -2376,7 +2426,8 @@ export async function crawlProject(opts: {
         // Generated ID-like path segments are usually fallback IDs or entity
         // records. They flood the catalog with duplicate page templates, so
         // skip them before screenshots or screen creation.
-        if (isGeneratedDynamicPath(pathname)) {
+        // Exception: seed path patterns declared by the caller bypass this guard.
+        if (isGeneratedDynamicPath(pathname) && !_activeSeedPatterns.some(re => re.test(pathname))) {
           console.log(`[crawler] skip generated dynamic route: ${url}`);
           continue;
         }
@@ -3231,10 +3282,20 @@ export async function crawlProject(opts: {
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
           // Note: preDomCount < aiMinElements guard removed — it was backwards: sparse pages (SPAs) need AI most.
           discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined)
+            .then((metas) => {
+              // Persist nav-graph edges fire-and-forget
+              if (metas.length > 0) {
+                AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, screen.id, metas.map((m) => ({
+                  href: m.url, text: m.text, landmark: m.landmark, linkType: 'NAV_LINK', source: m.source, jobId: opts.jobId,
+                }))).catch(() => {});
+              }
+              return metas.map(m => m.url);
+            })
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
-              const merged = (cachedLinks ? Array.from(new Set([...cachedLinks, ...freshLinks])) : freshLinks)
+              const freshUrls = freshLinks;
+              const merged = (cachedLinks ? Array.from(new Set([...cachedLinks, ...freshUrls])) : freshUrls)
                 .filter((href) => isUsefulNavigationUrl(href, origin));
               logger.info({ url, domLinks: freshLinks.length, cached: cachedLinks?.length ?? 0, merged: merged.length }, '[crawler] link-discovery: result');
               // Always persist fresh recipe so stale cache gets overwritten with new AI discoveries
@@ -3250,7 +3311,7 @@ export async function crawlProject(opts: {
               return merged;
             })
             .catch((err: any) => {
-              const fallback: string[] = err?.domLinks ?? [];
+              const fallback: string[] = (err?.domLinks ?? []).map((l: DiscoveredLinkMeta) => l.url);
               if (err?.isRateLimit) {
                 aiErrorStreak++;
                 if (aiErrorStreak >= 5) { aiSkipCount = 3; aiErrorStreak = 0; logger.warn('[crawler] 5 consecutive AI rate limits — pausing AI link discovery for next 3 screens then retrying'); }
@@ -3823,7 +3884,7 @@ export async function crawlProject(opts: {
               // Don't flood the queue with generated ID-like route segments.
               try {
                 const hrefPath = new URL(href).pathname;
-                if (isGeneratedDynamicPath(hrefPath)) continue;
+                if (isGeneratedDynamicPath(hrefPath) && !_activeSeedPatterns.some(re => re.test(hrefPath))) continue;
               } catch { /* ignore invalid URLs */ }
               if (robotsCache && !await robotsCache.isAllowed(href)) continue;
               queued.add(canonHref);
@@ -4062,7 +4123,14 @@ export async function crawlProject(opts: {
             }
             const [elems, links, linkMappings] = await Promise.all([
               extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
-              discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined),
+              discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined).then((metas) => {
+                if (metas.length > 0) {
+                  AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, screen.id, metas.map((m) => ({
+                    href: m.url, text: m.text, landmark: m.landmark, linkType: 'NAV_LINK', source: m.source, jobId: opts.jobId,
+                  }))).catch(() => {});
+                }
+                return metas.map(m => m.url);
+              }),
               extractLinkMappings(page, origin),
             ]);
             for (const el of elems) {
@@ -4077,7 +4145,8 @@ export async function crawlProject(opts: {
               pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
             }
             // Also enqueue links Skyvern page found for further ZeTa crawling
-            for (const href of links) {
+            for (const link of links) {
+              const href = link;
               const c = canonicalize(href);
               if (isUsefulNavigationUrl(href, origin) && !queued.has(c) && !visited.has(c)) { queued.add(c); queue.push(href); }
             }
@@ -4233,7 +4302,16 @@ export async function crawlProject(opts: {
             }
           }
           // Harvest any new links revealed by async content
-          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []);
+          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined)
+            .then((metas) => {
+              if (metas.length > 0) {
+                AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, existingScreenId, metas.map((m) => ({
+                  href: m.url, text: m.text, landmark: m.landmark, linkType: 'NAV_LINK', source: m.source, jobId: opts.jobId,
+                }))).catch(() => {});
+              }
+              return metas.map(m => m.url);
+            })
+            .catch(() => []);
           for (const href of revisitLinks) {
             const canon = canonicalize(href);
             if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
@@ -4285,7 +4363,16 @@ export async function crawlProject(opts: {
           }
           const [p3bElems, p3bLinks] = await Promise.all([
             extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
-            discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined).catch(() => []),
+            discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined)
+              .then((metas) => {
+                if (metas.length > 0) {
+                  AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, screen.id, metas.map((m) => ({
+                    href: m.url, text: m.text, landmark: m.landmark, linkType: 'NAV_LINK', source: m.source, jobId: opts.jobId,
+                  }))).catch(() => {});
+                }
+                return metas.map(m => m.url);
+              })
+              .catch(() => []),
           ]);
           if (p3bElems.length > 0) {
             await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
@@ -4369,7 +4456,16 @@ export async function crawlProject(opts: {
               }
               const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
                 extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
-                discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []),
+                discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined)
+                  .then((metas) => {
+                    if (metas.length > 0) {
+                      AppBrain.saveDiscoveredLinks(opts.tenantId, opts.projectId, gapScreen.id, metas.map((m) => ({
+                        href: m.url, text: m.text, landmark: m.landmark, linkType: 'NAV_LINK', source: m.source, jobId: opts.jobId,
+                      }))).catch(() => {});
+                    }
+                    return metas.map(m => m.url);
+                  })
+                  .catch(() => []),
                 extractLinkMappings(page, origin).catch(() => []),
               ]);
               for (const el of gapElems) {
@@ -4494,6 +4590,26 @@ export async function crawlProject(opts: {
       } catch { return false; }
     }).length;
     logger.info({ transitionCount }, 'KAG: wired element→screen transitions');
+
+    // Navigation graph: resolve DiscoveredLink.toScreenId by matching href → visited screen IDs.
+    // Build a raw-href → screenId map using both canonical and pathname-based lookups.
+    if (opts.jobId) {
+      const hrefToScreenId = new Map<string, string>();
+      for (const [canonUrl, screenId] of visited) {
+        if (screenId && screenId !== '_pending_' && screenId !== '_error_') {
+          hrefToScreenId.set(canonUrl, screenId);
+          try {
+            const u = new URL(canonUrl);
+            hrefToScreenId.set(u.href, screenId);
+            // Also index by origin+pathname so links with extra query params still resolve
+            hrefToScreenId.set(u.origin + u.pathname, screenId);
+            hrefToScreenId.set(u.origin + u.pathname.replace(/\/$/, ''), screenId);
+          } catch { /* skip */ }
+        }
+      }
+      AppBrain.resolveDiscoveredLinkTargets(opts.tenantId, opts.projectId, opts.jobId, hrefToScreenId).catch(() => {});
+    }
+
     AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
       jobId: opts.jobId,
       type: 'WORKFLOW_GRAPH_SUMMARY',
@@ -4672,7 +4788,7 @@ export async function crawlProject(opts: {
       const probeInputs = discovered
         .filter((d) => d.screenId)
         .map((d) => ({ url: d.url, screenId: d.screenId!, elements: d.elementFacts ?? [] }));
-      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
+      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress, opts.jobId);
     }
 
     // Step 7: End-of-crawl state graph coverage report.
@@ -4681,7 +4797,32 @@ export async function crawlProject(opts: {
     // loop above and are saved as PER_SCREEN_PROBES evidence.
     try {
       await onProgress?.(90, 'Generating state graph coverage report');
+      // Populate stateGraph from discovered screens + BFS edges (previously always empty).
       const stateGraph = new ScreenTransitionGraph();
+      const now = new Date().toISOString();
+      for (const d of discovered) {
+        if (!d.screenId || !d.url) continue;
+        const hashVal = simHash32(d.url);
+        stateGraph.addState({
+          id: d.screenId,
+          url: d.url,
+          normalizedUrl: d.url,
+          title: d.url,
+          interactiveElementCount: d.elements ?? 0,
+          domFingerprint: hashVal.toString(16).padStart(8, '0'),
+          simHashValue: hashVal,
+          visitedAt: now,
+        });
+      }
+      // Wire transitions from BFS edges
+      for (const edge of edges) {
+        const fromId = visited.get(canonicalize(edge.fromUrl));
+        const toId = visited.get(canonicalize(edge.toUrl));
+        if (fromId && toId && fromId !== '_pending_' && fromId !== '_error_' &&
+            toId !== '_pending_' && toId !== '_error_') {
+          stateGraph.addTransition(fromId, toId, 'navigate', edge.toUrl, now);
+        }
+      }
       const coveragePayload: Record<string, unknown> = {
         stateGraph: stateGraph.getCoverageReport(),
         totalScreensCrawled: discovered.filter(d => !!d.screenId).length,
@@ -4961,7 +5102,7 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
   }
 }
 
-async function discoverLinks(stagehand: Stagehand, origin: string, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, skipAi = false, rawPage?: any): Promise<string[]> {
+async function discoverLinks(stagehand: Stagehand, origin: string, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, skipAi = false, rawPage?: any): Promise<DiscoveredLinkMeta[]> {
   // Expand collapsed nav menus (hamburger, aria-haspopup, Radix closed triggers) so hidden routes
   // become part of the DOM before querying. This catches SPA nav dropdowns that only render on click.
   const navExpanded = await expandNavMenus(page);
@@ -4971,7 +5112,9 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
   const domLinks = await domFallbackLinks(page, origin, rawPage);
 
   // Interact with tabs, carousels, and accordions to reveal hidden content and discover more links.
-  const interactiveLinks = await interactAndCollectLinks(page, origin);
+  // interactAndCollectLinks returns string[] — convert to DiscoveredLinkMeta[] with source INTERACTIVE.
+  const interactiveRaw = await interactAndCollectLinks(page, origin);
+  const interactiveLinks: DiscoveredLinkMeta[] = interactiveRaw.map((url) => ({ url, source: 'INTERACTIVE' as const }));
 
   // Dismiss any opened menus so page is clean for screenshot/element extraction after this call.
   if (navExpanded) {
@@ -4984,7 +5127,15 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
     }).catch(() => { });
   }
 
-  const allDomLinks = Array.from(new Set([...domLinks, ...interactiveLinks]));
+  // Deduplicate by URL — DOM wins over INTERACTIVE for metadata richness.
+  const seenUrls = new Set<string>();
+  const allDomLinks: DiscoveredLinkMeta[] = [];
+  for (const link of [...domLinks, ...interactiveLinks]) {
+    if (!seenUrls.has(link.url)) {
+      seenUrls.add(link.url);
+      allDomLinks.push(link);
+    }
+  }
 
   if (allDomLinks.length >= 50 || skipAi) return allDomLinks;
 
@@ -5004,7 +5155,7 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
     if (!extracted) return allDomLinks;
     const { links } = extracted as any;
     const currentUrl: string = page.url?.() ?? '';
-    const aiLinks = links
+    const aiLinks: DiscoveredLinkMeta[] = links
       .map((href: string) => {
         try { const u = new URL(href, currentUrl); if (!/^#!?\//.test(u.hash)) u.hash = ''; return u.toString(); }
         catch { return null; }
@@ -5012,9 +5163,19 @@ async function discoverLinks(stagehand: Stagehand, origin: string, page: any, ll
       .filter((u: string | null): u is string => {
         if (!u) return false;
         return isUsefulNavigationUrl(u, origin);
-      });
-    // Always merge DOM + AI — AI complements, never replaces
-    return Array.from(new Set([...allDomLinks, ...aiLinks])).filter((href) => isUsefulNavigationUrl(href, origin));
+      })
+      .map((url: string) => ({ url, source: 'AI' as const }));
+
+    // Always merge DOM + AI — AI complements, never replaces. Dedup by URL.
+    const merged: DiscoveredLinkMeta[] = [...allDomLinks];
+    const mergedUrls = new Set(merged.map((l) => l.url));
+    for (const link of aiLinks) {
+      if (!mergedUrls.has(link.url) && isUsefulNavigationUrl(link.url, origin)) {
+        mergedUrls.add(link.url);
+        merged.push(link);
+      }
+    }
+    return merged.filter((l) => isUsefulNavigationUrl(l.url, origin));
   } catch (err: any) {
     // Detect OpenAI rate-limit errors — signal caller via thrown special error so circuit breaker can trip
     const is429 = err?.status === 429 || String(err?.message ?? '').includes('429') || String(err?.message ?? '').includes('rate limit');
@@ -6225,14 +6386,16 @@ async function expandNavMenus(page: any): Promise<boolean> {
   } catch { return false; }
 }
 
-async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promise<string[]> {
+async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promise<DiscoveredLinkMeta[]> {
   // Named function so .toString() gives the raw function body for CDP evaluation.
   // esbuild wraps nested named arrow-function constants with __name() — prepend a
   // tiny polyfill so the browser has it when we serialize this function as a string.
+  // IMPORTANT: _linkEval is serialized to a string via .toString() and evaluated in the
+  // browser via CDP. It CANNOT contain TypeScript types or Node.js imports.
   function _linkEval(originStr: string) {
     const logout = /logout|log out|sign out|delete/i;
     const isRouteHash = (hash: string) => /^#!?\//.test(hash);
-    const seen = new Set<string>();
+    const seen = new Map();
     const add = (rawHref: string, textContent: string) => {
       if (logout.test(textContent)) return;
       if (!rawHref) return;
@@ -6241,7 +6404,7 @@ async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promi
       try {
         const u = new URL(rawHref, location.href);
         if (!isRouteHash(u.hash)) u.hash = '';
-        if (new URL(u).origin === originStr) seen.add(u.toString());
+        if (new URL(u).origin === originStr) seen.set(u.toString(), (textContent || '').trim().slice(0, 120));
       } catch { /* ignore */ }
     };
     const allRoots: Array<Document | ShadowRoot> = [document];
@@ -6273,7 +6436,22 @@ async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promi
         });
       } catch (_) { /* element threw — skip root */ }
     }
-    return Array.from(seen);
+    return Array.from(seen.entries()).map(([url, text]) => ({ url, text }));
+  }
+
+  function inferLandmark(url: string): string | undefined {
+    const p = url.toLowerCase();
+    if (p.includes('/auth') || p.includes('/login') || p.includes('/register')) return 'nav';
+    return undefined;
+  }
+
+  function toMeta(raw: Array<{ url: string; text: string }>): DiscoveredLinkMeta[] {
+    return raw.map(({ url, text }) => ({
+      url,
+      text: text || undefined,
+      landmark: inferLandmark(url),
+      source: 'DOM' as const,
+    }));
   }
 
   const _linkEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_linkEval.toString()})(${JSON.stringify(origin)})})();`;
@@ -6291,7 +6469,7 @@ async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promi
             awaitPromise: false,
           });
           if (!evalResult?.exceptionDetails && Array.isArray(evalResult?.result?.value)) {
-            return evalResult.result.value as string[];
+            return toMeta(evalResult.result.value as Array<{ url: string; text: string }>);
           }
           if (evalResult?.exceptionDetails) {
             console.warn('[crawler] domFallbackLinks CDP error:', evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
@@ -6307,7 +6485,7 @@ async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promi
 
   // Fallback: pass string expression so esbuild __name wrapper never reaches browser
   const evalPage = rawPage ?? page;
-  return (evalPage.evaluate(_linkEvalStr as any) as Promise<string[]>).catch((err: any) => {
+  return (evalPage.evaluate(_linkEvalStr as any) as Promise<Array<{ url: string; text: string }>>).then(toMeta).catch((err: any) => {
     console.warn('[crawler] domFallbackLinks page.evaluate threw:', err?.message ?? String(err));
     return [];
   });

package/src/extractors/branding-extractor.ts

@@ -0,0 +1,159 @@
+import type { Page } from 'playwright';
+
+export interface BrandingColors {
+  primary?: string;
+  accent?: string;
+  background?: string;
+  textPrimary?: string;
+  link?: string;
+}
+
+export interface BrandingFont {
+  family: string;
+  role: 'heading' | 'body';
+}
+
+export interface BrandingImages {
+  logo?: string;
+  favicon?: string;
+  ogImage?: string;
+}
+
+export interface BrandingPersonality {
+  tone?: string;
+  energy?: 'high' | 'medium' | 'low';
+  targetAudience?: string;
+}
+
+export interface BrandingResult {
+  colorScheme: 'light' | 'dark';
+  colors: BrandingColors;
+  fonts: BrandingFont[];
+  images: BrandingImages;
+  personality?: BrandingPersonality;
+  designSystem?: { framework?: string; componentLibrary?: string };
+  metadata?: { title?: string; description?: string; themeColor?: string; language?: string };
+  confidence: { colors: number; fonts: number; images: number; overall: number };
+}
+
+export async function extractBranding(page: Page): Promise<BrandingResult> {
+  try {
+    const result = await page.evaluate(() => {
+      const bodyStyle = window.getComputedStyle(document.body);
+      const isDark = window.matchMedia('(prefers-color-scheme: dark)').matches ||
+        document.documentElement.getAttribute('data-theme') === 'dark' ||
+        document.body.classList.contains('dark');
+
+      // Extract colors from computed styles
+      const bg = bodyStyle.backgroundColor;
+      const text = bodyStyle.color;
+
+      // Find primary button color
+      const primaryBtn = document.querySelector(
+        'button[class*="primary"], .btn-primary, [class*="btn-primary"], [class*="button-primary"]'
+      );
+      const primaryColor = primaryBtn ? window.getComputedStyle(primaryBtn).backgroundColor : undefined;
+
+      // Find link color
+      const firstLink = document.querySelector('a');
+      const linkColor = firstLink ? window.getComputedStyle(firstLink).color : undefined;
+
+      // Extract font families
+      const headingEl = document.querySelector('h1, h2');
+      const bodyEl = document.body;
+      const headingFont = headingEl ? window.getComputedStyle(headingEl).fontFamily : bodyStyle.fontFamily;
+      const bodyFont = bodyStyle.fontFamily;
+
+      // Extract images
+      const logoEl = document.querySelector(
+        'header img, img[alt*="logo" i], img[class*="logo" i], [class*="logo"] img, [id*="logo"] img'
+      ) as HTMLImageElement | null;
+      const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]') as HTMLLinkElement | null;
+      const ogImageEl = document.querySelector('meta[property="og:image"]') as HTMLMetaElement | null;
+
+      // Extract metadata
+      const titleEl = document.querySelector('title');
+      const descEl = document.querySelector('meta[name="description"]') as HTMLMetaElement | null;
+      const themeColorEl = document.querySelector('meta[name="theme-color"]') as HTMLMetaElement | null;
+
+      // Detect design framework
+      const hasBootstrap = !!document.querySelector('[class*="btn btn-"], .container-fluid, .navbar-expand');
+      const hasTailwind = !!document.querySelector('[class*="flex "], [class*="text-"], [class*="bg-"]');
+      const hasChakra = !!(document as any).__chakra_factory__ || !!document.querySelector('[class*="chakra-"]');
+      const hasMUI = !!document.querySelector('.MuiButton-root, .MuiTypography-root');
+
+      const framework = hasBootstrap ? 'Bootstrap' : hasTailwind ? 'Tailwind CSS' : hasChakra ? 'Chakra UI' : hasMUI ? 'Material UI' : null;
+
+      return {
+        isDark,
+        bg,
+        text,
+        primaryColor,
+        linkColor,
+        headingFont,
+        bodyFont,
+        logo: logoEl?.src || logoEl?.getAttribute('src') || undefined,
+        favicon: faviconEl?.href || undefined,
+        ogImage: ogImageEl?.content || undefined,
+        title: titleEl?.textContent || undefined,
+        description: descEl?.content || undefined,
+        themeColor: themeColorEl?.content || undefined,
+        language: document.documentElement.lang || undefined,
+        framework,
+      };
+    });
+
+    const fonts: BrandingFont[] = [];
+    if (result.headingFont && result.headingFont !== result.bodyFont) {
+      fonts.push({ family: result.headingFont.split(',')[0].replace(/['"]/g, '').trim(), role: 'heading' });
+    }
+    if (result.bodyFont) {
+      fonts.push({ family: result.bodyFont.split(',')[0].replace(/['"]/g, '').trim(), role: 'body' });
+    }
+
+    const colorConfidence = (result.bg || result.text || result.primaryColor) ? 0.8 : 0.3;
+    const fontConfidence = fonts.length > 0 ? 0.7 : 0.2;
+    const imageConfidence = (result.logo || result.ogImage) ? 0.9 : 0.2;
+
+    return {
+      colorScheme: result.isDark ? 'dark' : 'light',
+      colors: {
+        background: result.bg || undefined,
+        textPrimary: result.text || undefined,
+        primary: result.primaryColor || undefined,
+        link: result.linkColor || undefined,
+      },
+      fonts,
+      images: {
+        logo: result.logo,
+        favicon: result.favicon,
+        ogImage: result.ogImage,
+      },
+      designSystem: {
+        framework: result.framework || undefined,
+        componentLibrary: undefined,
+      },
+      metadata: {
+        title: result.title,
+        description: result.description,
+        themeColor: result.themeColor,
+        language: result.language,
+      },
+      confidence: {
+        colors: colorConfidence,
+        fonts: fontConfidence,
+        images: imageConfidence,
+        overall: (colorConfidence + fontConfidence + imageConfidence) / 3,
+      },
+    };
+  } catch (err) {
+    console.warn('[branding-extractor] Extraction failed:', err);
+    return {
+      colorScheme: 'light',
+      colors: {},
+      fonts: [],
+      images: {},
+      confidence: { colors: 0, fonts: 0, images: 0, overall: 0 },
+    };
+  }
+}