@detiq/crawler 1.0.6 → 1.0.8

package/src/crawler.ts

@@ -20,7 +20,7 @@ import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 // Option 3: Camoufox — Firefox-based browser with built-in fingerprint randomisation.
 // Loaded dynamically (not at startup) to avoid ESM/CJS conflict when API imports crawler.
 import { spawn, execSync, type ChildProcess } from 'node:child_process';
-import { readFileSync, existsSync as existsSyncFs } from 'node:fs';
+import { existsSync as existsSyncFs, readFileSync } from 'node:fs';
 import { createRequire } from 'node:module';
 
 stealthChromium.use(StealthPlugin());
@@ -35,7 +35,7 @@ import type { Credentials } from './auth.js';
 import { skyvernFallback, skyvernNavigate, type SkyvernConfig } from './skyvern.js';
 import { loginWithStagehand } from './ai-login.js';
 import { readScreen } from './screen-reader.js';
-import { probeAllScreens } from './interactive-probe.js';
+import { probeAllScreens, probeKeyboardNav } from './interactive-probe.js';
 import { computeElementHash } from './element-hash.js';
 import { logger } from './logger.js';
 import { solveCaptcha } from './captcha-solver.js';
@@ -67,6 +67,7 @@ import { auditColorContrast } from './color-contrast.js';
 import { detectFeatureFlags } from './feature-flag-detector.js';
 import { detectLanguage } from './language-detector.js';
 import { detectForms } from './form-detector.js';
+import { auditAccessibility } from './accessibility-auditor.js';
 import { RedirectRegistry } from './redirect-tracker.js';
 import { DeadLinkTracker } from './dead-link-detector.js';
 import { screensToSitemapUrls, writeSitemap } from './sitemap-writer.js';
@@ -268,6 +269,7 @@ async function launchChromeCDP(
       '--remote-allow-origins=*',
       '--no-first-run',
       '--no-default-browser-check',
+      '--window-size=1280,900',
     ],
   });
 
@@ -1613,16 +1615,18 @@ export async function crawlProject(opts: {
     const performanceMetrics: PerformanceMetric[] = [];
     const seenDomHashes = new Set<string>(opts.knownDomHashes ?? []);
 
-    // Content/legal pages — still discovered as links but skipped from crawl queue
-    // (no test case value; pollute element counts with nav noise)
-    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+    // Legal/policy-only pages — no interactive elements worth testing; skip to prevent noise
+    // NOTE: /about, /contact, /careers, /help are intentionally NOT skipped — they have
+    // real test targets (contact forms, job apply forms, help search, team content)
+    const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
     const segIdx = opts.segmentIndex ?? 1;
 
     const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
     // AutoThrottle state — exponential moving average of navigation latency
     let avgNavMs = 0;
     let navCount = 0;
-    // Rate-limit circuit breaker: after 3 consecutive AI errors (429), skip AI for 10 screens
+    // Rate-limit circuit breaker: after 5 consecutive AI errors (429), skip AI for 3 screens then retry
+    // (was: 3 errors → skip 10 screens — too aggressive for Nvidia NIM which rate-limits quickly)
     let aiErrorStreak = 0;
     let aiSkipCount = 0;
     let aiRateLimitWarned = false;
@@ -2095,8 +2099,10 @@ export async function crawlProject(opts: {
           try {
             const _require = createRequire(import.meta.url);
             const axePath = _require.resolve('axe-core');
-            const axeScript = readFileSync(axePath, 'utf8');
-            await page.addScriptTag({ content: axeScript });
+            // page.evaluate() with script content bypasses CSP (DevTools Runtime.evaluate).
+            // page.addScriptTag({ path }) is blocked by strict-CSP on many production sites.
+            const axeContent = readFileSync(axePath, 'utf8');
+            await page.evaluate(axeContent);
             const result = await page.evaluate(async () => {
               return (window as any).axe.run(document, {
                 runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'best-practice'] },
@@ -2114,7 +2120,10 @@ export async function crawlProject(opts: {
             }));
             const criticalCount = violations.filter((v: any) => v.impact === 'critical').length;
             return { violations, violationCount: violations.length, criticalCount };
-          } catch { return null; }
+          } catch (err) {
+            logger.warn({ err: String((err as any)?.message ?? err) }, '[crawler] axe-core a11y audit failed (non-fatal)');
+            return null;
+          }
         })();
         const a11yScore = a11yResult
           ? Math.max(0, 100 - (a11yResult.criticalCount * 20) - ((a11yResult.violationCount - a11yResult.criticalCount) * 5))
@@ -2256,12 +2265,13 @@ export async function crawlProject(opts: {
         // screen is now declared; page still has current page state and is fully hydrated.
         void (async () => {
           const screenId = screen.id;
-          const [seoR, contrastR, flagsR, langR, formsR] = await Promise.allSettled([
+          const [seoR, contrastR, flagsR, langR, formsR, a11yManualR] = await Promise.allSettled([
             checkSeo(page, url),
             auditColorContrast(page, url),
             detectFeatureFlags(page, url),
             detectLanguage(page, url),
             detectForms(page),
+            auditAccessibility(page, url),
           ]);
           const base = { screenId, jobId: opts.jobId };
           if (seoR.status === 'fulfilled')
@@ -2274,10 +2284,20 @@ export async function crawlProject(opts: {
             AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'LANGUAGE_DETECTION', payload: langR.value }).catch(() => { });
           if (formsR.status === 'fulfilled' && Array.isArray(formsR.value) && formsR.value.length)
             AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'FORMS_DETECTED', payload: { forms: formsR.value } }).catch(() => { });
+          if (a11yManualR.status === 'fulfilled' && (a11yManualR.value as any).violations?.length)
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'A11Y_MANUAL_AUDIT', payload: a11yManualR.value }).catch(() => { });
         })();
 
         // Freeze all CSS animations/transitions so screenshots are stable across runs
-        try { await page.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; }' }); } catch { /* Stagehand proxy may not expose addStyleTag */ }
+        try { await page.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; animation-play-state: paused !important; animation-fill-mode: both !important; }' }); } catch { /* Stagehand proxy may not expose addStyleTag */ }
+        // Freeze JS-driven animations (requestAnimationFrame loops, setInterval animations)
+        await page.evaluate(() => {
+          try {
+            const noop = () => 0;
+            Object.defineProperty(window, 'requestAnimationFrame', { value: noop, configurable: true });
+            Object.defineProperty(window, 'cancelAnimationFrame', { value: noop, configurable: true });
+          } catch { /* property may not be configurable */ }
+        }).catch(() => { });
 
         // Replay-first: check for a cached link-discovery recipe before calling AI Stagehand
         const canonUrl = canonicalize(url);
@@ -2353,7 +2373,7 @@ export async function crawlProject(opts: {
               const fallback: string[] = err?.domLinks ?? [];
               if (err?.isRateLimit) {
                 aiErrorStreak++;
-                if (aiErrorStreak >= 3) { aiSkipCount = 10; logger.warn('[crawler] 3 consecutive AI rate limits — skipping AI link discovery for next 10 screens'); }
+                if (aiErrorStreak >= 5) { aiSkipCount = 3; aiErrorStreak = 0; logger.warn('[crawler] 5 consecutive AI rate limits — pausing AI link discovery for next 3 screens then retrying'); }
                 if (!aiRateLimitWarned) {
                   aiRateLimitWarned = true;
                   onProgress?.(20, 'AI quota exhausted — falling back to DOM-only link discovery').catch(() => { });
@@ -2365,6 +2385,14 @@ export async function crawlProject(opts: {
         ]);
         logger.info({ url, screenId: screen.id, version: screen.version, elements: elements.length, links: nextLinks.length, atSample: atElements.slice(0, 3).map((a: any) => a.label) }, '[crawler] screen captured');
         console.log('[crawler] AT:', JSON.stringify(atElements.slice(0, 5)));
+        // Fire-and-forget keyboard-nav probe — records Tab sequence and focus-indicator issues per screen
+        probeKeyboardNav(page).then((kbResult) => {
+          if (kbResult.focusableCount > 0) {
+            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+              screenId: screen.id, jobId: opts.jobId, type: 'KEYBOARD_NAV', payload: kbResult,
+            }).catch(() => { });
+          }
+        }).catch(() => { });
 
         // Capture DOM snapshot for structural analysis, and extract Markdown/text
         // content from the same DOM (crawl4AI-parity text output) — no extra fetch.
@@ -2412,11 +2440,13 @@ export async function crawlProject(opts: {
                 try {
                   const llmCfg = await getCachedCrawlLLM(opts.tenantId);
                   const snippet = extracted.markdown.slice(0, 1200);
+                  // Skip LLM when content is too thin — output would be useless
+                  if (snippet.trim().length < 80) return undefined;
                   const { text, usage } = await askTracked(
-                    'You are a QA analyst. Given page content, write ONE crisp sentence describing what this screen does or shows. Be specific. No preamble.',
+                    'You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.',
                     `Screen: "${name}"\nURL: ${url}\n\nContent:\n${snippet}`,
                     llmCfg,
-                    80,
+                    200,
                   );
                   if (usage) {
                     recordLLMCall({
@@ -2428,7 +2458,10 @@ export async function crawlProject(opts: {
                       usage,
                     }).catch(() => { });
                   }
-                  return text.trim().replace(/^["']|["']$/g, '');
+                  // Keep only the first sentence if the LLM returned multiple
+                  const clean = text.trim().replace(/^["']|["']$/g, '');
+                  const firstSentence = clean.match(/^[^.!?]+[.!?]/)?.[0] ?? clean;
+                  return firstSentence.trim();
                 } catch { return undefined; }
               })();
 
@@ -2633,6 +2666,8 @@ export async function crawlProject(opts: {
         }
 
         const linkMappings = await extractLinkMappings(page, origin);
+        // Clear existing elements before re-saving — prevents duplicates on re-crawl
+        await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
         // Batch element saves: 4 concurrent DB writes instead of serial round-trips
         for (let _ei = 0; _ei < elements.length; _ei += 4) {
           const _batch = elements.slice(_ei, _ei + 4);
@@ -3375,9 +3410,9 @@ export async function crawlProject(opts: {
       }
     }
 
-    // Step 6: Interactive probe — observe real dynamic behaviour (optional, ~30s extra)
-    // Auto-enable for sparse crawls (< 30 screens): interaction-gated states are proportionally more common.
-    const shouldRunInteractiveProbe = opts.interactive || discovered.filter(d => d.screenId).length < 30;
+    // Step 6: Interactive probe — always-on (C-06) to capture real validation errors, toasts, and modals.
+    // Previously gated on opts.interactive or sparse crawls; now unconditional for accurate expected-result generation.
+    const shouldRunInteractiveProbe = true;
     if (shouldRunInteractiveProbe) {
       await onProgress?.(88, 'Running interactive probe (capturing real errors & modals)');
       const probeInputs = discovered
@@ -3512,7 +3547,10 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
   // llmCfg reserved for per-task model selection (future: create task-scoped Stagehand)
   // DOM-first: fast, no LLM cost, covers server-rendered and most SPAs.
   // Threshold >= 2 so a lone dark-mode button doesn't suppress AI fallback for rich pages.
-  const domElements = await domFallbackElements(page);
+  const domElements = await domFallbackElements(page).catch((err: any) => {
+    console.warn('[crawler] domFallbackElements failed (page context issue or DOM error):', err?.message ?? err);
+    return [] as Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>;
+  });
 
   // FM-8: extract interactive elements from visible same-origin iframes
   const iframeElements: typeof domElements = [];
@@ -3553,12 +3591,40 @@ async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provi
     } catch { dynamicElements = []; }
   }
 
-  if (domElements.length >= 2) return [...domElements, ...iframeElements, ...dynamicElements];
-
-  // Content fallback: static/marketing pages have no interactive widgets but have headings,
-  // images, links, and meta tags that feed SEO/a11y/visual test generation.
+  // Always capture content elements (headings, meta, cards, nav, FAQ) regardless of interactive count.
+  // Previously this was fallback-only; now both paths run together so static sections are
+  // always visible to the test generator even on pages with forms.
   const contentElements = await contentFallbackElements(page);
-  if (contentElements.length > 0) return [...domElements, ...contentElements];
+
+  // Cross-source text deduplication: strips wrapper phrases like "Button to expand '...' details"
+  // and "... FAQ button" so duplicate FAQ buttons from DOM + content sources are collapsed.
+  const crossSourceDedup = <T extends { meaning: string; role: string }>(arr: T[]): T[] => {
+    const stripPhrases = (s: string) => s
+      .replace(/^button to expand ['"]?|['"]?\s*details$/gi, '')
+      .replace(/\s+(faq\s*)?button$/i, '')
+      .replace(/^button:\s*/i, '')
+      .toLowerCase().trim().slice(0, 80);
+    const ROLE_PRIORITY: Record<string, number> = { 'faq-question': 10, 'faq-answer': 9, 'h1': 8, 'h2': 7, 'h3': 6, 'h4': 5, 'heading': 8, button: 3, a: 2 };
+    const normMap = new Map<string, number>();
+    const result: T[] = [];
+    for (const el of arr) {
+      const norm = stripPhrases(el.meaning);
+      if (!norm) { result.push(el); continue; }
+      const existingIdx = normMap.get(norm);
+      if (existingIdx === undefined) { normMap.set(norm, result.length); result.push(el); }
+      else {
+        const exPri = ROLE_PRIORITY[result[existingIdx].role] ?? 1;
+        const thisPri = ROLE_PRIORITY[el.role] ?? 1;
+        if (thisPri > exPri) result[existingIdx] = el;
+      }
+    }
+    return result;
+  };
+
+  if (domElements.length >= 2) return crossSourceDedup([...domElements, ...iframeElements, ...dynamicElements, ...contentElements]);
+
+  // Static/marketing page — no interactive widgets; use content elements.
+  if (contentElements.length > 0) return crossSourceDedup([...domElements, ...contentElements]);
 
   // AI fallback: DOM found nothing or only noise — try AI (pure SPA, shadow DOM, canvas-based).
   try {
@@ -3654,17 +3720,20 @@ async function domFallbackElements(page: any): Promise<Array<{ meaning: string;
     const noise = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;
 
     const querySelectorAllDeep = (root: any, selector: string): Element[] => {
-      const results: Element[] = Array.from(root.querySelectorAll(selector));
-      root.querySelectorAll('*').forEach((el: any) => {
-        if (el.shadowRoot) {
-          results.push(...querySelectorAllDeep(el.shadowRoot, selector));
-        }
-      });
+      const results: Element[] = [];
+      const queue: any[] = [root];
+      while (queue.length > 0) {
+        const current = queue.shift();
+        Array.from(current.querySelectorAll(selector)).forEach((el: any) => results.push(el));
+        current.querySelectorAll('*').forEach((el: any) => {
+          if (el.shadowRoot) queue.push(el.shadowRoot);
+        });
+      }
       return results;
     };
 
-    // Expanded selector with more ARIA roles + rich text editors
-    const selector = 'input:not([type=hidden]),select,textarea,button,[role="button"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],a[role="button"],[contenteditable="true"]';
+    // Expanded selector — all interactive ARIA roles + native form elements + rich editors
+    const selector = 'input:not([type=hidden]),select,textarea,button,[role="button"],[role="checkbox"],[role="radio"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],[role="gridcell"],[role="link"],a[role="button"],[contenteditable="true"]';
 
     querySelectorAllDeep(document, selector).forEach((el: any) => {
       const style = window.getComputedStyle(el);
@@ -3673,7 +3742,11 @@ async function domFallbackElements(page: any): Promise<Array<{ meaning: string;
       if (el.hasAttribute('inert') || el.getAttribute('aria-disabled') === 'true') return;
 
       const rect = el.getBoundingClientRect();
-      if (rect.width === 0 && rect.height === 0) return;
+      const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));
+      if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;
+
+      // Skip elements inside 3rd-party chat widgets, cookie banners, floating overlays — these appear on EVERY page
+      if (el.closest('[id*="intercom"],[class*="intercom-"],[id*="crisp"],[class*="crisp-"],[id*="freshchat"],[id*="freshdesk"],[id*="drift"],[class*="drift-"],[id*="hubspot-"],[class*="hs-chat"],[class*="hs-widget"],[id*="tawk"],[class*="tawk-"],[id*="chat-widget"],[class*="chat-widget"],[class*="chatbot"],[id*="zopim"],[id*="ze-snippet"],[class*="cookie-banner"],[class*="cookie-consent"],[class*="gdpr-banner"],[id*="cookie-law"],[class*="cookie-law"],[id*="popup-form"],[class*="popup-form"],[class*="floating-form"],[class*="sticky-cta"],[id*="wm-ipp"],[class*="wm-ipp"]')) return;
 
       const tag = el.tagName.toLowerCase();
       const ariaRole = el.getAttribute('role');
@@ -3867,22 +3940,27 @@ async function domFallbackElements(page: any): Promise<Array<{ meaning: string;
       ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea', 'url', 'file'].includes(e.role) ? 0 : 1;
     out.sort((a, b) => priority(a) - priority(b));
     return out; // Cap removed
-  }).catch(() => []);
+  }).then((elems: any[]) => elems.map((e: any) => ({ ...e, isInteractive: true }))).catch(() => []);
 }
 
-/** For pages with no interactive elements (marketing/static pages), capture content anatomy:
- *  headings, images-with-alt, CTA links outside nav/header/footer, and meta tags.
- *  These give the test generator something to work with for SEO, a11y, and visual tests. */
+/** Capture content anatomy for EVERY page — not just static ones.
+ *  Headings, meta tags, cards, chips, FAQ items, nav/footer structure, section text.
+ *  Previously fallback-only; now always runs alongside domFallbackElements so
+ *  the test generator sees ALL page content, not just interactive widgets. */
 async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string }>> {
   return page.evaluate(() => {
     const out: { meaning: string; role: string; notes?: string }[] = [];
     const seen = new Set<string>();
     const addUniq = (item: { meaning: string; role: string; notes?: string }) => {
       const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
-      if (!seen.has(key) && item.meaning.trim().length > 0) { seen.add(key); out.push(item); }
+      if (!seen.has(key) && item.meaning.trim().length > 1) { seen.add(key); out.push(item); }
+    };
+    const isVisible = (el: Element) => {
+      const s = window.getComputedStyle(el);
+      return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0' && (el as HTMLElement).offsetParent !== null;
     };
 
-    // 1. Page-level meta (title, description, og:)
+    // 1. Page-level meta (title, description, og:, canonical)
     const title = document.title?.trim();
     if (title) addUniq({ meaning: title, role: 'page-title', notes: JSON.stringify({ tag: 'title' }) });
     const desc = document.querySelector('meta[name="description"]')?.getAttribute('content')?.trim();
@@ -3892,40 +3970,678 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href')?.trim();
     if (canonical) addUniq({ meaning: canonical, role: 'canonical-url', notes: JSON.stringify({ tag: 'link[rel=canonical]' }) });
 
-    // 2. Headings — page structure
-    document.querySelectorAll('h1, h2, h3').forEach((h: any) => {
-      const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80);
+    // 2. All headings h1–h6 — full page structure
+    document.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h: any) => {
+      if (!isVisible(h)) return;
+      const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
       if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) });
     });
 
-    // 3. Images with alt text — accessibility targets
-    document.querySelectorAll('img[alt]').forEach((img: any) => {
-      const alt = img.getAttribute('alt')?.trim().slice(0, 80);
-      const src = img.getAttribute('src')?.slice(0, 120);
-      if (alt) addUniq({ meaning: alt, role: 'image', notes: JSON.stringify({ src }) });
+    // 3. FAQ / accordion — <details>/<summary>, ARIA pattern, and class-name scan
+    document.querySelectorAll('details, [data-state="closed"], [data-state="open"]').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const summary = el.tagName === 'DETAILS'
+        ? el.querySelector('summary')
+        : el.querySelector('[class*="trigger" i], [class*="header" i], [class*="question" i], [class*="title" i]') ?? el;
+      const qText = summary?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120);
+      if (qText && qText.length > 3) {
+        addUniq({ meaning: qText, role: 'faq-question', notes: JSON.stringify({ expanded: el.hasAttribute('open') || el.getAttribute('data-state') === 'open' }) });
+      }
+      // Answer: for <details> it's the non-summary child; for React accordions try both child content and next sibling
+      const contentEl = el.tagName === 'DETAILS'
+        ? Array.from(el.children).find((c: any) => c.tagName !== 'SUMMARY')
+        : (el.querySelector('[class*="content" i], [class*="answer" i], [class*="panel" i], [class*="body" i]:not(body)') ?? el.nextElementSibling);
+      if (contentEl) {
+        const aText = (contentEl as any).textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
+        if (aText && aText.length > 5 && aText !== qText) addUniq({ meaning: aText, role: 'faq-answer', notes: JSON.stringify({ question: qText }) });
+      }
+    });
+    // 3b. FAQ by class-name container scan — catches sites without ARIA or data-state attributes
+    document.querySelectorAll('[class*="faq" i]:not(script):not(style):not(nav), [id*="faq" i]:not(script):not(style)').forEach((container: any) => {
+      if (!isVisible(container)) return;
+      const qEls = container.querySelectorAll('[class*="question" i], [class*="trigger" i], summary, dt, [class*="heading" i]');
+      const aEls = container.querySelectorAll('[class*="answer" i], [class*="content" i]:not(main):not(article), [class*="panel" i], dd');
+      qEls.forEach((q: any) => {
+        const qText = q.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120);
+        if (qText && qText.length > 3) addUniq({ meaning: qText, role: 'faq-question', notes: JSON.stringify({ source: 'class-faq' }) });
+      });
+      aEls.forEach((a: any) => {
+        const aText = a.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
+        if (aText && aText.length > 5) addUniq({ meaning: aText, role: 'faq-answer', notes: JSON.stringify({ source: 'class-faq' }) });
+      });
     });
 
-    // 4. CTA / content links — outside nav, header, footer
+    // 4. Card and chip content (component-library agnostic)
+    document.querySelectorAll('[class*="card" i]:not(nav):not(header):not(footer), [class*="chip" i], [class*="badge" i], [class*="tag" i], [class*="pill" i]').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
+      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' });
+    });
+
+    // 5. Feature / service list items inside main content (capped per category)
+    const mainContent = document.querySelector('main, [role="main"], article') ?? document.body;
+    let listItemCount = 0;
+    mainContent.querySelectorAll('li').forEach((li: any) => {
+      if (listItemCount >= 30) return;
+      if (!isVisible(li)) return;
+      const text = li.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
+      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }); listItemCount++; }
+    });
+
+    // 6. First paragraph in each section (key descriptive copy)
+    mainContent.querySelectorAll('section > p:first-of-type, article > p:first-of-type, [class*="section" i] > p:first-of-type').forEach((p: any) => {
+      if (!isVisible(p)) return;
+      const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 250);
+      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' });
+    });
+
+    // 7. CTA section text (standalone paragraphs near buttons that aren't form fields)
+    document.querySelectorAll('[class*="cta" i] p, [class*="call-to-action" i] p, section:not(form) > p').forEach((p: any) => {
+      if (!isVisible(p)) return;
+      const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 300);
+      if (text && text.length > 10 && text.length < 400) addUniq({ meaning: text, role: 'cta-section' });
+    });
+
+    // 8. Navigation links — explicitly captured (excluded from interactive element selector)
+    // Use a lighter visibility check: skip offsetParent because nav is often inside a responsive
+    // container (e.g. hidden lg:grid) that may be display:none at the crawler's viewport,
+    // causing offsetParent to be null even though the link exists in the page's nav structure.
+    const isNavVisible = (el: Element) => {
+      const s = window.getComputedStyle(el);
+      return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
+    };
+    document.querySelectorAll('nav a, [role="navigation"] a, header a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a: any) => {
+      if (!isNavVisible(a)) return;
+      const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+      const href = a.href;
+      if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
+      // Walk up to find nearest nav/header/footer ancestor to get landmark label
+      let landmark = 'Site Navigation';
+      let el: any = a.parentElement;
+      while (el) {
+        const tag = el.tagName?.toLowerCase();
+        const role = el.getAttribute?.('role')?.toLowerCase();
+        if (tag === 'header' || role === 'banner') { landmark = 'Header'; break; }
+        if (tag === 'footer' || role === 'contentinfo') { landmark = 'Footer'; break; }
+        if (tag === 'nav' || role === 'navigation') {
+          const directLabel = el.getAttribute('aria-label')?.trim() ?? '';
+          const labelId = el.getAttribute('aria-labelledby');
+          const referencedLabel = labelId ? (document.getElementById(labelId)?.textContent?.trim() ?? '') : '';
+          landmark = directLabel || referencedLabel || 'Site Navigation';
+          break;
+        }
+        el = el.parentElement;
+      }
+      addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark }) });
+    });
+
+    // 9. Breadcrumb items
+    document.querySelectorAll('[aria-label*="breadcrumb" i] a, [class*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] a').forEach((a: any) => {
+      const label = a.textContent?.replace(/\s+/g, ' ').trim().slice(0, 60);
+      const href = a.href;
+      if (label) addUniq({ meaning: label, role: 'breadcrumb-item', notes: JSON.stringify({ href: href?.slice(0, 150) }) });
+    });
+
+    // 10. Footer links and contact info (capped at 25)
+    // Matches semantic <footer>, ARIA contentinfo, and common div/section footer patterns
+    let footerLinkCount = 0;
+    const footerEls = new Set<Element>();
+    document.querySelectorAll('footer, [role="contentinfo"], [class*="footer" i]:not(script):not(style):not(nav), [id*="footer" i]:not(script):not(style)').forEach((f: any) => footerEls.add(f));
+    const footerLinks: Element[] = [];
+    footerEls.forEach((f) => f.querySelectorAll('a').forEach((a: Element) => footerLinks.push(a)));
+    // Deduplicate links that appear in multiple matched footer containers
+    const seenFooterHrefs = new Set<string>();
+    footerLinks.filter((a: any) => {
+      const href = a.href ?? '';
+      if (seenFooterHrefs.has(href)) return false;
+      seenFooterHrefs.add(href);
+      return true;
+    }).forEach((a: any) => {
+      if (footerLinkCount >= 25) return;
+      if (!isVisible(a)) return;
+      const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
+      const href = a.href ?? '';
+      if (!label) return;
+      if (/^mailto:/i.test(href)) {
+        addUniq({ meaning: label || href, role: 'footer-contact', notes: JSON.stringify({ href }) }); footerLinkCount++;
+      } else if (/^tel:/i.test(href)) {
+        addUniq({ meaning: label || href, role: 'footer-contact', notes: JSON.stringify({ href }) }); footerLinkCount++;
+      } else if (label.length > 0) {
+        addUniq({ meaning: label, role: 'footer-link', notes: JSON.stringify({ href: href.slice(0, 150) }) }); footerLinkCount++;
+      }
+    });
+
+    // 11. All image types — img tags (with or without alt), picture/source, CSS background-image, lazy-load patterns
+    let imgCount = 0;
+    // Lighter visibility check for images: skip only display:none or visibility:hidden.
+    // Do NOT check offsetParent (fails for position:fixed heroes) or opacity (kills fade-in animations).
+    const isImgVisible = (el: any) => {
+      const s = window.getComputedStyle(el);
+      return s.display !== 'none' && s.visibility !== 'hidden';
+    };
+    // 11a. <img> — both with and without alt (capped at 20)
+    document.querySelectorAll('img').forEach((img: any) => {
+      if (imgCount >= 20) return;
+      if (!isImgVisible(img)) return;
+      const alt = img.getAttribute('alt')?.trim().slice(0, 80) ?? '';
+      const src = (img.getAttribute('src') || img.getAttribute('data-src') || img.getAttribute('data-lazy-src') || img.getAttribute('data-original') || '').slice(0, 120);
+      // Also grab first URL from srcset (Next.js Image component uses srcset directly on <img>)
+      const srcsetFirst = img.getAttribute('srcset')?.split(',')[0]?.trim().split(' ')[0]?.slice(0, 120) ?? '';
+      const resolvedSrc = src || srcsetFirst;
+      if (!resolvedSrc && !alt) return;
+      const label = alt || resolvedSrc.split('/').pop()?.split('?')[0]?.slice(0, 60) || 'Image';
+      addUniq({ meaning: label, role: alt ? 'image' : 'image-no-alt', notes: JSON.stringify({ src: resolvedSrc, alt, hasAlt: !!alt }) });
+      imgCount++;
+    });
+    // 11b. <picture> / <source srcset> — responsive images
+    document.querySelectorAll('picture source[srcset]').forEach((src: any) => {
+      if (imgCount >= 25) return;
+      const srcset = src.getAttribute('srcset')?.split(',')[0]?.trim().split(' ')[0]?.slice(0, 120) ?? '';
+      const media = src.getAttribute('media') ?? '';
+      if (srcset) { addUniq({ meaning: `Responsive image${media ? ' (' + media + ')' : ''}: ${srcset.split('/').pop()?.split('?')[0] ?? ''}`, role: 'picture-source', notes: JSON.stringify({ srcset: srcset.slice(0, 150), media }) }); imgCount++; }
+    });
+    // 11c. CSS background-image via inline style and computed style (capped at 15)
+    let bgCount = 0;
+    document.querySelectorAll('[style*="background-image"], [style*="background:"]').forEach((el: any) => {
+      if (bgCount >= 15) return;
+      if (!isImgVisible(el)) return;
+      const style = el.getAttribute('style') ?? '';
+      const match = style.match(/url\(['"]?([^'")\s]+)['"]?\)/i);
+      if (match) {
+        const url = match[1].slice(0, 120);
+        const role = el.tagName.toLowerCase();
+        const context = (el.getAttribute('aria-label') || el.getAttribute('title') || el.className.split(' ').find((c: string) => c.length > 3 && !['true','false'].includes(c)) || '').slice(0, 60);
+        addUniq({ meaning: context || `BG image on <${role}>: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, element: role, context }) });
+        bgCount++;
+      }
+    });
+    // 11d. Computed background-image on key visual elements (expanded class list)
+    document.querySelectorAll('[class*="hero" i], [class*="banner" i], [class*="cover" i], [class*="header-bg" i], [class*="card" i]:not(nav), [class*="feature" i], [class*="service" i], [class*="about" i], [class*="team" i], section').forEach((el: any) => {
+      if (bgCount >= 15) return;
+      if (!isImgVisible(el)) return;
+      try {
+        const computed = window.getComputedStyle(el).backgroundImage;
+        if (computed && computed !== 'none' && computed.includes('url(')) {
+          const match = computed.match(/url\(['"]?([^'")\s]+)['"]?\)/i);
+          if (match) {
+            const url = match[1].slice(0, 120);
+            const tag = el.tagName.toLowerCase();
+            const cls = Array.from(el.classList as DOMTokenList).find((c: any) => typeof c === 'string' && c.length > 3 && !/^(true|false|active|visible)$/.test(c)) ?? '';
+            addUniq({ meaning: `CSS bg image on .${cls || tag}: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, cssClass: cls, element: tag }) });
+            bgCount++;
+          }
+        }
+      } catch { /* getComputedStyle not always available */ }
+    });
+    // 11e. Lazy-load data attribute patterns (data-bg, data-background, data-lazy)
+    document.querySelectorAll('[data-bg], [data-background], [data-background-image]').forEach((el: any) => {
+      if (bgCount >= 15) return;
+      if (!isVisible(el)) return;
+      const url = (el.getAttribute('data-bg') || el.getAttribute('data-background') || el.getAttribute('data-background-image') || '').slice(0, 120);
+      if (url) { addUniq({ meaning: `Lazy BG image: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, pattern: 'data-bg' }) }); bgCount++; }
+    });
+
+    // 12. Content links outside nav/header/footer (capped at 20)
     const navTags = new Set(['nav', 'header', 'footer']);
+    let contentLinkCount = 0;
     document.querySelectorAll('a[href]').forEach((a: any) => {
+      if (contentLinkCount >= 20) return;
+      if (!isVisible(a)) return;
       let p = a.parentElement;
       while (p) { if (navTags.has(p.tagName?.toLowerCase())) return; p = p.parentElement; }
       const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 80);
       const href = a.href;
-      if (label && href && !/^(#|javascript:|mailto:)/i.test(href)) {
+      if (label && href && !/^(#$|javascript:|mailto:|tel:)/i.test(href)) {
         const isExternal = !href.includes(location.hostname);
         addUniq({ meaning: label, role: isExternal ? 'external-link' : 'internal-link', notes: JSON.stringify({ href: href.slice(0, 120) }) });
+        contentLinkCount++;
       }
     });
 
-    // 5. Animated / interactive content markers (CSS animation, video, canvas)
+    // 13. Animated / interactive content markers
     const hasAnimation = !!document.querySelector('[class*="animate"], [class*="motion"], [class*="transition"], [class*="scroll-"], [data-aos], [data-scroll]');
     if (hasAnimation) addUniq({ meaning: 'CSS animations / scroll effects detected on page', role: 'animation-layer', notes: JSON.stringify({ detected: true }) });
     if (document.querySelector('video')) addUniq({ meaning: 'Video element present', role: 'video', notes: JSON.stringify({ count: document.querySelectorAll('video').length }) });
     if (document.querySelector('canvas')) addUniq({ meaning: 'Canvas element present', role: 'canvas' });
 
-    return out.slice(0, 30);
-  }).catch(() => []);
+    // C-07: Section structure as named semantic units
+    document.querySelectorAll('main section, article section, [role="main"] section').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const heading = el.querySelector('h1,h2,h3,h4')?.textContent?.trim() ?? '';
+      const itemCount = el.querySelectorAll('li, [class*="card" i], [class*="chip" i]').length;
+      const paragraphs = el.querySelectorAll('p').length;
+      const meaning = heading || `Section (${itemCount} items)`;
+      if (meaning) addUniq({ meaning, role: 'page-section', notes: JSON.stringify({ heading, itemCount, paragraphs, id: el.id || el.getAttribute('data-section') }) });
+    });
+
+    // 14. Extended meta tags (og:description, og:image, twitter card)
+    const ogDesc = document.querySelector('meta[property="og:description"]')?.getAttribute('content')?.trim();
+    if (ogDesc) addUniq({ meaning: ogDesc, role: 'og-description', notes: JSON.stringify({ tag: 'meta[property=og:description]' }) });
+    const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content')?.trim();
+    if (ogImage) addUniq({ meaning: ogImage, role: 'og-image', notes: JSON.stringify({ tag: 'meta[property=og:image]', src: ogImage }) });
+    const twitterTitle = document.querySelector('meta[name="twitter:title"]')?.getAttribute('content')?.trim();
+    if (twitterTitle) addUniq({ meaning: twitterTitle, role: 'twitter-title', notes: JSON.stringify({ tag: 'meta[name=twitter:title]' }) });
+    const twitterDesc = document.querySelector('meta[name="twitter:description"]')?.getAttribute('content')?.trim();
+    if (twitterDesc) addUniq({ meaning: twitterDesc, role: 'twitter-description', notes: JSON.stringify({ tag: 'meta[name=twitter:description]' }) });
+    const jsonFonts: string[] = [];
+    document.querySelectorAll('link[rel="preload"][as="font"], link[href*="fonts.googleapis"]').forEach((l: any) => {
+      const url = l.href?.slice(0, 120);
+      if (url) jsonFonts.push(url);
+    });
+    if (jsonFonts.length) addUniq({ meaning: `Font imports: ${jsonFonts.length} font file(s)`, role: 'font-stack', notes: JSON.stringify({ fonts: jsonFonts.slice(0, 5) }) });
+
+    // 15. JSON-LD structured data — extract FAQ answers, deliverables, tech stack, metrics
+    document.querySelectorAll('script[type="application/ld+json"], script[class*="tsr"]').forEach((s: any) => {
+      try {
+        const data = JSON.parse(s.textContent ?? '{}');
+        const items = Array.isArray(data) ? data : [data];
+        for (const item of items) {
+          // FAQ answers from JSON-LD (often NOT rendered in collapsed accordions)
+          const faqArr = item.mainEntity ?? item.faqs ?? item.questions ?? item.faq ?? [];
+          (Array.isArray(faqArr) ? faqArr : []).forEach((f: any) => {
+            const q = f.name ?? f.question ?? '';
+            const a = f.acceptedAnswer?.text ?? f.answer ?? '';
+            if (q) addUniq({ meaning: String(q).slice(0, 150), role: 'faq-question', notes: JSON.stringify({ source: 'json-ld' }) });
+            if (a) addUniq({ meaning: String(a).slice(0, 200), role: 'faq-answer', notes: JSON.stringify({ question: String(q).slice(0, 80), source: 'json-ld' }) });
+          });
+          // Deliverables with descriptions
+          const deliverables = item.deliverables ?? item.services ?? item.offerings ?? [];
+          (Array.isArray(deliverables) ? deliverables : []).forEach((d: any) => {
+            const title = d.title ?? d.name ?? '';
+            const desc = d.description ?? d.desc ?? '';
+            if (title) addUniq({ meaning: `${title}${desc ? ': ' + desc.slice(0, 120) : ''}`, role: 'deliverable', notes: JSON.stringify({ title, description: String(desc).slice(0, 200) }) });
+          });
+          // Tech stack items
+          const stack = item.stack ?? item.technologies ?? item.tools ?? [];
+          (Array.isArray(stack) ? stack : []).forEach((tool: any) => {
+            const name = typeof tool === 'string' ? tool : (tool.name ?? '');
+            if (name) addUniq({ meaning: name, role: 'tech-stack-item', notes: JSON.stringify({ source: 'json-ld' }) });
+          });
+          // Outcome/metric cards
+          const outcomes = item.outcomes ?? item.metrics ?? item.stats ?? [];
+          (Array.isArray(outcomes) ? outcomes : []).forEach((m: any) => {
+            const metric = m.metric ?? m.value ?? '';
+            const label = m.label ?? m.name ?? '';
+            if (metric && label) addUniq({ meaning: `${metric} — ${label}`, role: 'metric-card', notes: JSON.stringify({ metric, label }) });
+          });
+          // Features list
+          const features = item.features ?? item.capabilities ?? [];
+          (Array.isArray(features) ? features : []).forEach((f: any) => {
+            const name = typeof f === 'string' ? f : (f.name ?? '');
+            if (name) addUniq({ meaning: name, role: 'feature-badge', notes: JSON.stringify({ source: 'json-ld' }) });
+          });
+          // HowTo steps (tutorials, guides, recipes)
+          if (item['@type'] === 'HowTo' || item['@type'] === 'Recipe') {
+            const steps = item.step ?? item.steps ?? [];
+            (Array.isArray(steps) ? steps : []).forEach((step: any, idx: number) => {
+              const text = step.text ?? step.description ?? (typeof step === 'string' ? step : '');
+              const name = step.name ?? `Step ${idx + 1}`;
+              if (text) addUniq({ meaning: `${name}: ${String(text).slice(0, 200)}`, role: 'howto-step', notes: JSON.stringify({ stepIndex: idx + 1, name }) });
+            });
+          }
+          // BreadcrumbList — page hierarchy for navigation test generation
+          if (item['@type'] === 'BreadcrumbList') {
+            const crumbs = item.itemListElement ?? [];
+            (Array.isArray(crumbs) ? crumbs : []).forEach((crumb: any) => {
+              const name = crumb.name ?? crumb.item?.name ?? '';
+              const url = crumb.item?.['@id'] ?? crumb.item ?? '';
+              if (name) addUniq({ meaning: name, role: 'breadcrumb-item', notes: JSON.stringify({ position: crumb.position, url }) });
+            });
+          }
+          // Event details (conference, webinar, workshop)
+          if (item['@type'] === 'Event') {
+            const name = item.name ?? '';
+            const startDate = item.startDate ?? '';
+            const location = item.location?.name ?? item.location?.address?.streetAddress ?? '';
+            if (name) addUniq({ meaning: name, role: 'event-name', notes: JSON.stringify({ startDate, location, source: 'json-ld' }) });
+          }
+          // JobPosting — for job boards and career pages
+          if (item['@type'] === 'JobPosting') {
+            const title = item.title ?? '';
+            const desc = item.description ?? '';
+            const employer = item.hiringOrganization?.name ?? '';
+            if (title) addUniq({ meaning: `${title}${employer ? ' at ' + employer : ''}`, role: 'job-posting', notes: JSON.stringify({ description: String(desc).slice(0, 200) }) });
+          }
+          // Review / AggregateRating — for product and service pages
+          if (item['@type'] === 'Review') {
+            const body = item.reviewBody ?? '';
+            const rating = item.reviewRating?.ratingValue ?? '';
+            const author = item.author?.name ?? '';
+            if (body) addUniq({ meaning: String(body).slice(0, 200), role: 'review-text', notes: JSON.stringify({ rating, author }) });
+          }
+        }
+      } catch { /* non-fatal — malformed JSON-LD */ }
+    });
+
+    // 16. Tab panels — capture visible and aria-labeled panels (process steps, tabbed content)
+    document.querySelectorAll('[role="tabpanel"], [data-state="active"][id*="panel"], [id*="panel"][aria-labelledby]').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const heading = el.querySelector('h2, h3, h4')?.textContent?.trim() ?? '';
+      const desc = el.querySelector('p')?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200) ?? '';
+      const badges = Array.from(el.querySelectorAll('[class*="badge" i], [class*="tag" i], [class*="pill" i], [class*="chip" i], [class*="rounded-full" i]')).map((b: any) => b.textContent?.trim()).filter(Boolean).slice(0, 8);
+      const meaning = heading || desc;
+      if (meaning) addUniq({ meaning: meaning.slice(0, 150), role: 'tab-panel', notes: JSON.stringify({ heading, description: desc, badges, panelId: el.id }) });
+    });
+
+    // 17. FAQ accordion: aria-controls pattern — find answer panel via aria-controls attribute
+    // Include both aria-expanded buttons and plain aria-controls (some FAQ components omit aria-expanded)
+    document.querySelectorAll('[aria-controls]').forEach((btn: any) => {
+      if (!isVisible(btn)) return;
+      const panelId = btn.getAttribute('aria-controls');
+      if (!panelId) return;
+      const panel = document.getElementById(panelId);
+      if (!panel) return;
+      const qText = btn.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120);
+      // Read panel text even if hidden (accordion may be collapsed)
+      const aText = panel.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
+      if (qText && qText.length > 3) addUniq({ meaning: qText, role: 'faq-question', notes: JSON.stringify({ panelId, expanded: btn.getAttribute('aria-expanded') }) });
+      if (aText && aText.length > 5 && aText !== qText) addUniq({ meaning: aText, role: 'faq-answer', notes: JSON.stringify({ question: qText?.slice(0, 80), source: 'aria-controls' }) });
+    });
+
+    // 18. Tech stack / tool pills — monospace pill elements outside nav/footer
+    const mainEl = document.querySelector('main, [role="main"], article') ?? document.body;
+    mainEl.querySelectorAll('[class*="mono" i]:not(code):not(pre), [class*="pill" i]:not(nav), [class*="chip" i]:not(nav), [class*="tag" i]:not(nav), [class*="badge" i]:not(nav)').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 60);
+      if (text && text.length > 1 && text.length < 60 && !text.includes(' ')) {
+        // Single-word or hyphenated tool name
+        addUniq({ meaning: text, role: 'tech-stack-item', notes: JSON.stringify({ source: 'dom-pill' }) });
+      } else if (text && text.length > 1 && text.length < 40) {
+        addUniq({ meaning: text, role: 'feature-badge', notes: JSON.stringify({ source: 'dom-pill' }) });
+      }
+    });
+
+    // 19. Related/linked service cards in main content (cards with links to other pages)
+    mainEl.querySelectorAll('a[href][class*="card" i], [class*="card" i] a[href], [class*="service" i][href], a[class*="service" i]').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const anchor = el.tagName === 'A' ? el : el.querySelector('a');
+      if (!anchor) return;
+      const href = anchor.href;
+      if (!href || /^(#|javascript:|mailto:|tel:)/i.test(href)) return;
+      const heading = el.querySelector('h2, h3, h4')?.textContent?.trim() ?? '';
+      const desc = el.querySelector('p')?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120) ?? '';
+      const label = anchor.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80) ?? '';
+      const meaning = heading || label || 'Linked card';
+      if (meaning && meaning !== 'Linked card' || (heading && desc)) {
+        addUniq({ meaning: `${heading || meaning}${desc ? ': ' + desc : ''}`.slice(0, 180), role: 'service-card', notes: JSON.stringify({ href: href.slice(0, 150), heading, description: desc }) });
+      }
+    });
+
+    // 20. Metric/outcome cards — structured numeric data cards
+    mainEl.querySelectorAll('[class*="metric" i], [class*="outcome" i], [class*="stat" i], [class*="kpi" i]').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
+      if (text && text.length > 2 && text.length < 120) addUniq({ meaning: text, role: 'metric-card', notes: JSON.stringify({ source: 'dom-class' }) });
+    });
+
+    // 21. Screen-reader-only headings (sr-only) — accessibility landmarks hidden from visual view
+    document.querySelectorAll('.sr-only, [class*="sr-only" i], [class*="visually-hidden" i]').forEach((el: any) => {
+      const tag = el.tagName?.toLowerCase() ?? '';
+      if (!/^h[1-6]$/.test(tag)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
+      if (text) addUniq({ meaning: text, role: `${tag}-sr-only`, notes: JSON.stringify({ srOnly: true, tag }) });
+    });
+
+    // 22. CSS custom property (design token) detection
+    // Step 1: enumerate :root / html rules from all same-origin stylesheets
+    try {
+      const cssTokens: Record<string, string> = {};
+      const rootStyle = window.getComputedStyle(document.documentElement);
+      // Enumerate from stylesheets (catches ALL custom properties, not just a hardcoded list)
+      try {
+        for (let si = 0; si < document.styleSheets.length; si++) {
+          try {
+            const sheet = document.styleSheets[si];
+            const rules = sheet.cssRules;
+            for (let ri = 0; ri < rules.length; ri++) {
+              const rule = rules[ri] as any;
+              if (rule.selectorText === ':root' || rule.selectorText === 'html' || rule.selectorText === ':root, html') {
+                const style = rule.style;
+                for (let pi = 0; pi < style.length; pi++) {
+                  const prop = style[pi];
+                  if (prop.startsWith('--') && Object.keys(cssTokens).length < 30) {
+                    const val = style.getPropertyValue(prop).trim();
+                    if (val) cssTokens[prop] = val.slice(0, 60);
+                  }
+                }
+              }
+            }
+          } catch { /* cross-origin sheet, skip */ }
+        }
+      } catch { /* no styleSheets access */ }
+      // Step 2: fall back to computed value probe with common framework naming schemes
+      // Covers: plain CSS, Tailwind v4, Bootstrap, Material UI, Shadcn/ui, custom brands
+      const probePrefixes = ['', '--color-', '--bs-', '--brand-', '--clr-', '--theme-', '--md-sys-color-'];
+      const probeNames = ['primary', 'secondary', 'accent', 'background', 'foreground', 'surface', 'text', 'muted', 'border', 'ring', 'card', 'destructive', 'warning', 'success'];
+      const probeFonts = ['--font-sans', '--font-mono', '--font-display', '--font-heading', '--font-body', '--font-family-base'];
+      for (const pfx of probePrefixes) {
+        for (const name of probeNames) {
+          const v = `${pfx}${name}`;
+          if (!cssTokens[v]) {
+            const val = rootStyle.getPropertyValue(v).trim();
+            if (val) cssTokens[v] = val.slice(0, 60);
+          }
+        }
+      }
+      for (const v of probeFonts) {
+        if (!cssTokens[v]) {
+          const val = rootStyle.getPropertyValue(v).trim();
+          if (val) cssTokens[v] = val.slice(0, 60);
+        }
+      }
+      if (Object.keys(cssTokens).length > 0) {
+        const limited = Object.fromEntries(Object.entries(cssTokens).slice(0, 25));
+        addUniq({ meaning: `Design tokens: ${Object.keys(limited).join(', ')}`, role: 'design-tokens', notes: JSON.stringify(limited) });
+      }
+    } catch { /* computed styles not available */ }
+
+    // 23. Data tables — headers + row count (critical for business/admin pages)
+    document.querySelectorAll('table').forEach((table: any) => {
+      if (!isVisible(table)) return;
+      const headers = Array.from(table.querySelectorAll('thead th, thead td, tr:first-child th')).map((th: any) => th.textContent?.trim()).filter(Boolean).slice(0, 8);
+      const rowCount = table.querySelectorAll('tbody tr').length;
+      if (headers.length > 0) addUniq({ meaning: `Table: ${headers.join(' · ')}`, role: 'data-table', notes: JSON.stringify({ headers, rowCount }) });
+    });
+
+    // 24. Definition lists — term/description pairs (used for specs, glossaries, FAQs, metadata)
+    document.querySelectorAll('dl').forEach((dl: any) => {
+      if (!isVisible(dl)) return;
+      const terms = Array.from(dl.querySelectorAll('dt')).map((dt: any) => dt.textContent?.trim()).filter(Boolean).slice(0, 10);
+      if (terms.length > 0) addUniq({ meaning: `Definition list: ${terms.slice(0, 4).join(', ')}`, role: 'definition-list', notes: JSON.stringify({ terms: terms.slice(0, 10) }) });
+    });
+
+    // 25. Blockquotes / testimonials / reviews / pull quotes
+    let quoteCount = 0;
+    document.querySelectorAll('blockquote, [class*="testimonial" i], [class*="review" i]:not(script), [class*="quote" i]:not([class*="icon"]), [class*="pullquote" i]').forEach((el: any) => {
+      if (quoteCount >= 10) return;
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
+      const author = el.querySelector('[class*="author" i], [class*="name" i], cite')?.textContent?.trim().slice(0, 60) ?? '';
+      if (text && text.length > 20) { addUniq({ meaning: `${text.slice(0, 160)}${author ? ' — ' + author : ''}`, role: 'testimonial', notes: JSON.stringify({ author }) }); quoteCount++; }
+    });
+
+    // 26. Alert / notice / notification banners (non-interactive, but key content for test assertions)
+    document.querySelectorAll('[role="alert"], [role="status"], [role="banner"]:not(header), .alert, .notice, .notification, .warning-banner, .error-banner, .success-banner, [class*="alert-" i]:not(button), [class*="toast" i]:not(button), [class*="snackbar" i]').forEach((el: any) => {
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
+      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) });
+    });
+
+    // 27. Pricing / plan cards — structured pricing content
+    let pricingCount = 0;
+    document.querySelectorAll('[class*="pricing" i], [class*="plan" i]:not(main):not(article), [class*="tier" i]:not(nav), [class*="package" i]:not(script)').forEach((el: any) => {
+      if (pricingCount >= 8) return;
+      if (!isVisible(el)) return;
+      const name = el.querySelector('h2,h3,h4,[class*="plan-name" i],[class*="tier-name" i]')?.textContent?.trim().slice(0, 60) ?? '';
+      const price = el.querySelector('[class*="price" i],[class*="amount" i]')?.textContent?.trim().slice(0, 40) ?? '';
+      if (name || price) { addUniq({ meaning: `${name || 'Plan'}${price ? ': ' + price : ''}`, role: 'pricing-plan', notes: JSON.stringify({ planName: name, price }) }); pricingCount++; }
+    });
+
+    // 28. Pagination — page navigation for multi-page content
+    const paginationEl = document.querySelector('[role="navigation"][aria-label*="paginat" i], [class*="pagination" i], [aria-label*="paginat" i]');
+    if (paginationEl && isVisible(paginationEl)) {
+      const pages = Array.from(paginationEl.querySelectorAll('a, button')).map((p: any) => p.textContent?.trim()).filter(Boolean).slice(0, 10);
+      if (pages.length > 0) addUniq({ meaning: `Pagination: ${pages.join(', ')}`, role: 'pagination', notes: JSON.stringify({ pages }) });
+    }
+
+    // 29. Code blocks / pre blocks — critical for developer documentation pages
+    let codeCount = 0;
+    document.querySelectorAll('pre code, [class*="code-block" i], [class*="highlight" i] pre').forEach((el: any) => {
+      if (codeCount >= 5) return;
+      if (!isVisible(el)) return;
+      const lang = el.getAttribute('class')?.match(/language-(\w+)/)?.[1] ?? '';
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
+      if (text && text.length > 10) { addUniq({ meaning: `Code example${lang ? ' (' + lang + ')' : ''}: ${text.slice(0, 80)}`, role: 'code-block', notes: JSON.stringify({ language: lang }) }); codeCount++; }
+    });
+
+    // 30. Dropdown / select menu items exposed via ARIA listbox/option
+    let dropdownCount = 0;
+    document.querySelectorAll('[role="listbox"] [role="option"], [role="option"]:not([role="listbox"] *)').forEach((el: any) => {
+      if (dropdownCount >= 20) return;
+      if (!isVisible(el)) return;
+      const text = el.textContent?.trim().slice(0, 80);
+      if (text) { addUniq({ meaning: text, role: 'dropdown-option', notes: JSON.stringify({ selected: el.getAttribute('aria-selected') }) }); dropdownCount++; }
+    });
+
+    // 31. Timeline / process steps (common on landing pages)
+    let timelineCount = 0;
+    document.querySelectorAll('[class*="timeline" i] li, [class*="timeline" i] [class*="step" i], [class*="process" i] li, [class*="steps" i] li').forEach((el: any) => {
+      if (timelineCount >= 10) return;
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
+      if (text && text.length > 5) { addUniq({ meaning: text, role: 'timeline-step', notes: JSON.stringify({ source: 'dom-class' }) }); timelineCount++; }
+    });
+
+    // 32. Team members / person cards — social proof and about pages
+    let teamCount = 0;
+    document.querySelectorAll('[class*="team-member" i], [class*="team-card" i], [class*="person-card" i], [class*="bio" i][class*="card" i]').forEach((el: any) => {
+      if (teamCount >= 10) return;
+      if (!isVisible(el)) return;
+      const name = el.querySelector('[class*="name" i], h3, h4')?.textContent?.trim().slice(0, 60) ?? '';
+      const role = el.querySelector('[class*="role" i], [class*="title" i], [class*="position" i]')?.textContent?.trim().slice(0, 60) ?? '';
+      if (name) { addUniq({ meaning: `${name}${role ? ' — ' + role : ''}`, role: 'team-member', notes: JSON.stringify({ name, role }) }); teamCount++; }
+    });
+
+    // 33. Video embeds — YouTube, Vimeo, Wistia, Loom iframes (src is readable even cross-origin)
+    document.querySelectorAll('iframe').forEach((iframe: any) => {
+      const src = iframe.getAttribute('src') || iframe.getAttribute('data-src') || '';
+      const title = iframe.getAttribute('title') || iframe.getAttribute('aria-label') || '';
+      if (!src) return;
+      if (/youtube\.com\/embed|youtu\.be\/|youtube-nocookie\.com/i.test(src)) {
+        const videoId = src.match(/\/embed\/([^/?&]+)/)?.[1] ?? '';
+        addUniq({ meaning: title || `YouTube video${videoId ? ' (ID: ' + videoId + ')' : ''}`, role: 'video-embed', notes: JSON.stringify({ platform: 'youtube', src: src.slice(0, 200), title, videoId }) });
+      } else if (/vimeo\.com\/video\//i.test(src)) {
+        const videoId = src.match(/video\/(\d+)/)?.[1] ?? '';
+        addUniq({ meaning: title || `Vimeo video${videoId ? ' (ID: ' + videoId + ')' : ''}`, role: 'video-embed', notes: JSON.stringify({ platform: 'vimeo', src: src.slice(0, 200), title, videoId }) });
+      } else if (/wistia\.com|loom\.com|dailymotion\.com|twitch\.tv|brightcove\.com/i.test(src)) {
+        const platform = /wistia/.test(src) ? 'wistia' : /loom/.test(src) ? 'loom' : /dailymotion/.test(src) ? 'dailymotion' : /twitch/.test(src) ? 'twitch' : 'brightcove';
+        addUniq({ meaning: title || `${platform} video`, role: 'video-embed', notes: JSON.stringify({ platform, src: src.slice(0, 200), title }) });
+      } else if (/maps\.google\.com|google\.com\/maps\/embed/i.test(src)) {
+        const q = new URLSearchParams(src.split('?')[1] ?? '').get('q') ?? new URLSearchParams(src.split('?')[1] ?? '').get('query') ?? '';
+        addUniq({ meaning: title || `Google Maps${q ? ': ' + decodeURIComponent(q).slice(0, 80) : ''}`, role: 'map-embed', notes: JSON.stringify({ provider: 'google', src: src.slice(0, 200), query: q }) });
+      } else if (/openstreetmap\.org|mapbox\.com|here\.com|maps\.apple\.com/i.test(src)) {
+        const provider = /openstreetmap/.test(src) ? 'OpenStreetMap' : /mapbox/.test(src) ? 'Mapbox' : /here\.com/.test(src) ? 'HERE Maps' : 'Apple Maps';
+        addUniq({ meaning: title || `${provider} embed`, role: 'map-embed', notes: JSON.stringify({ provider, src: src.slice(0, 200), title }) });
+      }
+    });
+
+    // 34. SVG content — only meaningful SVGs (have title/aria-label AND are large enough to be illustrations)
+    document.querySelectorAll('svg').forEach((svg: any) => {
+      if (!isVisible(svg)) return;
+      const titleEl = svg.querySelector('title');
+      const label = svg.getAttribute('aria-label')?.trim() || titleEl?.textContent?.trim() || '';
+      if (!label) return;
+      const rect = svg.getBoundingClientRect();
+      if (rect.width < 48 && rect.height < 48) return; // Skip icon-size SVGs
+      addUniq({ meaning: label.slice(0, 100), role: 'svg-image', notes: JSON.stringify({ width: Math.round(rect.width), height: Math.round(rect.height) }) });
+    });
+
+    // 35. Animated counters — read data-target/data-count BEFORE animation completes
+    let counterCount = 0;
+    document.querySelectorAll('[data-count],[data-target],[data-to],[data-countup],[class*="counter" i]:not(button),[class*="count-up" i],[class*="odometer" i]').forEach((el: any) => {
+      if (counterCount >= 15) return;
+      if (!isVisible(el)) return;
+      const targetVal = el.getAttribute('data-count') || el.getAttribute('data-target') || el.getAttribute('data-to') || el.getAttribute('data-countup') || '';
+      const currentText = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 40) ?? '';
+      const contextLabel = el.closest('[class*="stat" i],[class*="metric" i],[class*="kpi" i],[class*="counter-item" i]')?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80) ?? '';
+      const value = targetVal || currentText;
+      if (value && value !== '0') {
+        addUniq({ meaning: contextLabel.slice(0, 100) || `Counter: ${value}`, role: 'animated-counter', notes: JSON.stringify({ targetValue: targetVal, displayedValue: currentText }) });
+        counterCount++;
+      }
+    });
+
+    // 36. Client / partner logos — distinguish from generic images
+    let logoCount = 0;
+    document.querySelectorAll('[class*="client" i] img,[class*="partner" i] img,[class*="logo-grid" i] img,[class*="clients-" i] img,[class*="brand-logo" i] img,[class*="trust-logo" i] img,[class*="sponsor" i] img').forEach((img: any) => {
+      if (logoCount >= 20) return;
+      if (!isVisible(img)) return;
+      const alt = img.getAttribute('alt')?.trim().slice(0, 60) ?? '';
+      const src = (img.getAttribute('src') || img.getAttribute('data-src') || '').slice(0, 120);
+      if (!alt && !src) return;
+      addUniq({ meaning: alt || src.split('/').pop()?.split('?')[0]?.slice(0, 50) || 'Client logo', role: 'client-logo', notes: JSON.stringify({ alt, src, hasAlt: !!alt }) });
+      logoCount++;
+    });
+
+    // 37. Awards / certifications / trust badges
+    let awardCount = 0;
+    document.querySelectorAll('[class*="award" i],[class*="certif" i]:not(script):not(link),[class*="accredit" i],[class*="trust-badge" i],[class*="recognition" i],[class*="badge" i][class*="award" i]').forEach((el: any) => {
+      if (awardCount >= 10) return;
+      if (!isVisible(el)) return;
+      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80);
+      const imgEl = el.querySelector('img');
+      const label = text || imgEl?.getAttribute('alt')?.trim() || '';
+      if (label && label.length > 2) { addUniq({ meaning: label, role: 'award-badge', notes: JSON.stringify({ hasImage: !!imgEl }) }); awardCount++; }
+    });
+
+    // 38. A/B testing detection — captures active experiments (not variant content)
+    try {
+      const abTools: string[] = [];
+      if ((window as any).optimizely || (window as any).Optimizely) abTools.push('Optimizely');
+      if ((window as any).VWO || (window as any)._vwo_code) abTools.push('VWO');
+      if ((window as any).__ABTasty || (window as any).ABTasty) abTools.push('AB Tasty');
+      if (document.querySelector('script[src*="optimize.js"]')) abTools.push('Google Optimize');
+      if ((window as any).splitbee) abTools.push('Splitbee');
+      if ((window as any).__GROWTHBOOK__ || document.querySelector('script[src*="growthbook"]')) abTools.push('GrowthBook');
+      if ((window as any).Kameleoon) abTools.push('Kameleoon');
+      if (abTools.length > 0) {
+        const optExp = (window as any).optimizely?.get?.('state')?.getActiveExperimentIds?.() ?? [];
+        addUniq({ meaning: `A/B testing: ${abTools.join(', ')}${optExp.length ? ' (' + optExp.length + ' active experiments)' : ''}`, role: 'ab-test-layer', notes: JSON.stringify({ tools: abTools, activeExperiments: optExp.slice(0, 10) }) });
+      }
+    } catch { /* best-effort */ }
+
+    // 39. Geo / personalization detection
+    try {
+      const signals: string[] = [];
+      if ((window as any).__geo || (window as any).__country || (window as any).__region) signals.push('window geo globals');
+      if (document.querySelector('[data-geo],[data-country],[data-region],[data-locale]')) signals.push('data-geo DOM attributes');
+      const ipScripts = Array.from(document.querySelectorAll('script[src]')).filter((s: any) => /ipinfo|maxmind|ipstack|ipgeolocation/i.test(s.getAttribute('src') ?? '')).length;
+      if (ipScripts > 0) signals.push('IP geolocation script');
+      if ((window as any).Intl?.DateTimeFormat?.()?.resolvedOptions?.()?.timeZone) signals.push(`timezone: ${(window as any).Intl.DateTimeFormat().resolvedOptions().timeZone}`);
+      if (signals.length > 1) {
+        addUniq({ meaning: `Personalization/geo signals: ${signals.join('; ')}`, role: 'personalization-layer', notes: JSON.stringify({ signals }) });
+      }
+    } catch { /* best-effort */ }
+
+    // 40. Hover-revealed content — read DOM text of CSS-hidden children on hover triggers
+    let hoverCount = 0;
+    document.querySelectorAll('[class*="group"]:not(nav):not(ul):not(ol),[class*="hover-trigger" i],[data-hover],[class*="has-dropdown" i]').forEach((trigger: any) => {
+      if (hoverCount >= 8) return;
+      if (!isVisible(trigger)) return;
+      const hiddenChildren = Array.from(trigger.querySelectorAll('[class*="group-hover"],[class*="hover-content" i],[class*="dropdown-menu" i],[class*="submenu" i]')).filter((c: any) => {
+        try { const s = window.getComputedStyle(c); return s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1; } catch { return false; }
+      });
+      for (const child of hiddenChildren.slice(0, 3)) {
+        const text = (child as any).textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
+        if (text && text.length > 3) {
+          addUniq({ meaning: text, role: 'hover-content', notes: JSON.stringify({ triggerClass: (trigger.className?.toString() ?? '').slice(0, 60) }) });
+          hoverCount++;
+        }
+      }
+    });
+
+    return out.slice(0, 400); // Per-category caps above; raised global cap to 400 for new categories
+  }).then((elems: any[]) => elems.map((e: any) => ({ ...e, isInteractive: false }))).catch(() => []);
 }
 
 /**