autokap 1.0.7 → 1.0.8

package/dist/browser.js

@@ -1,6 +1,7 @@
 import { chromium } from 'playwright';
 import sharp from 'sharp';
 import { createHash } from 'crypto';
+import { buildAKNodeRuntimeIndex, deriveInteractiveElementsFromAKTree, disambiguateFingerprint, focusAKTree, fingerprintAKNode, serializeAKTree, } from './ak-tree.js';
 /**
  * Set-of-Marks (SoM) annotation: overlays colored [N] badges on each visible
  * interactive element so the vision model can reference elements by their badge index.
@@ -95,8 +96,9 @@ function resolveEffectivePadding(config, bbox) {
         left = config.paddingLeft;
     return { top, right, bottom, left };
 }
-import { dismissCookiesAndWidgets } from './cookie-dismiss.js';
+import { dismissCookiesAndWidgets, ensureCaptureHideStyles } from './cookie-dismiss.js';
 import { CHROMIUM_ARGS, browserPool } from './browser-pool.js';
+import { logger } from './logger.js';
 async function withHelperTimeout(label, timeoutMs, work) {
     if (!timeoutMs || timeoutMs <= 0) {
         return work();
@@ -272,6 +274,22 @@ export function describeObservationChange(before, after) {
         && !changes.some((entry) => entry.startsWith('URL ->'))) {
         changes.push(`visible text changed to "${after.textSample.slice(0, 80)}"`);
     }
+    if (after.overlaySurface !== before.overlaySurface
+        && !changes.some((entry) => entry.startsWith('visible text changed'))) {
+        changes.push(after.overlaySurface
+            ? `overlay surface -> "${after.overlaySurface.slice(0, 80)}"`
+            : 'overlay surface cleared');
+    }
+    if (after.configurationSurface !== before.configurationSurface
+        && !changes.some((entry) => entry.startsWith('visible text changed'))) {
+        changes.push(after.configurationSurface
+            ? `configuration surface -> "${after.configurationSurface.slice(0, 80)}"`
+            : 'configuration surface cleared');
+    }
+    if (after.strongSurfaceSignature !== before.strongSurfaceSignature
+        && !changes.some((entry) => entry.startsWith('visible text changed'))) {
+        changes.push('content surface changed');
+    }
     return {
         before,
         after,
@@ -287,6 +305,7 @@ export class Browser {
     context = null;
     page = null;
     elementMap = new Map();
+    akNodeIndex = new Map();
     poolContext = false;
     constructor(options) {
         this.options = options;
@@ -324,7 +343,7 @@ export class Browser {
         // Dedicated browser process for video — cannot use the pool because
         // `recordVideo` must be set at context creation time.
         instance.browser = await chromium.launch({
-            headless: true,
+            headless: !options.headed,
             args: CHROMIUM_ARGS,
         });
         instance.context = await instance.browser.newContext({
@@ -360,14 +379,54 @@ export class Browser {
     }
     async launch() {
         // Pool mode: context and page are already initialized via fromPool()
-        if (this.poolContext)
+        if (this.poolContext) {
+            this.attachDebugLifecycleListeners();
             return;
+        }
         this.browser = await chromium.launch({
             headless: !this.options.headed,
             args: CHROMIUM_ARGS,
         });
         this.context = await this.browser.newContext(this.buildContextOptions());
         this.page = await this.context.newPage();
+        this.attachDebugLifecycleListeners();
+    }
+    /** Wires debug-only lifecycle listeners on the page/context/browser to surface
+     *  navigations, console errors, page crashes, and unexpected closures.
+     *  No-op when debug logging is disabled. */
+    attachDebugLifecycleListeners() {
+        if (!this.page || !this.context)
+            return;
+        const page = this.page;
+        const context = this.context;
+        page.on('crash', () => {
+            logger.debug(`[page] CRASH on ${page.url()}`);
+        });
+        page.on('close', () => {
+            logger.debug(`[page] CLOSE event on ${page.url()}`);
+        });
+        page.on('framenavigated', (frame) => {
+            if (frame === page.mainFrame()) {
+                logger.debug(`[page] navigated to ${frame.url()}`);
+            }
+        });
+        page.on('pageerror', (err) => {
+            logger.debug(`[page] pageerror: ${err.message.split('\n')[0]}`);
+        });
+        page.on('console', (msg) => {
+            const type = msg.type();
+            if (type === 'error' || type === 'warning') {
+                logger.debug(`[page] console.${type}: ${msg.text().slice(0, 200)}`);
+            }
+        });
+        context.on('close', () => {
+            logger.debug(`[context] CLOSE event`);
+        });
+        if (this.browser) {
+            this.browser.on('disconnected', () => {
+                logger.debug(`[browser] DISCONNECTED`);
+            });
+        }
     }
     async recreateContext(options = {}) {
         if (this.poolContext) {
@@ -456,10 +515,42 @@ export class Browser {
     }
     async navigateTo(url) {
         const page = this.ensurePage();
+        // Reject empty / whitespace URLs early — without this guard the
+        // `'https://' + url` fallback below produces "https://" which Playwright
+        // accepts but then crashes on with a confusing "Cannot navigate to invalid URL".
+        const trimmedUrl = (url ?? '').trim();
+        if (!trimmedUrl) {
+            throw new Error('Browser.navigateTo: empty URL. Check that all `{{loginUrl}}` / `{{email}}` ' +
+                'placeholders in your NAVIGATE opcodes are backed by valid credentials on the preset.');
+        }
+        url = trimmedUrl;
         // Fix common URL typos (htpps, hhtps, htps, etc.)
         url = url.replace(/^h+t+p+s?:\/\//i, (m) => /s/i.test(m) ? 'https://' : 'http://');
         if (!/^https?:\/\//i.test(url)) {
-            url = 'https://' + url;
+            // Resolve relative URLs (e.g. "/home", "dashboard") against the current page origin
+            if (url.startsWith('/')) {
+                const currentUrl = page.url();
+                try {
+                    const origin = new URL(currentUrl).origin;
+                    url = `${origin}${url}`;
+                }
+                catch {
+                    url = 'https://' + url;
+                }
+            }
+            else {
+                url = 'https://' + url;
+            }
+        }
+        // Final sanity check: must have a host after all the auto-prepend logic.
+        try {
+            const parsed = new URL(url);
+            if (!parsed.host) {
+                throw new Error(`Browser.navigateTo: resolved URL "${url}" has no host`);
+            }
+        }
+        catch (err) {
+            throw new Error(`Browser.navigateTo: invalid URL "${url}" — ${err instanceof Error ? err.message : String(err)}`);
         }
         try {
             await page.goto(url, { waitUntil: 'load', timeout: 20000 });
@@ -504,31 +595,26 @@ export class Browser {
         const page = this.ensurePage();
         // Move cursor off-screen to avoid hover effects in screenshots
         await page.mouse.move(0, 0);
+        await ensureCaptureHideStyles(page);
         return Buffer.from(await page.screenshot({ type: 'png', fullPage: false }));
     }
     async takeScreenshotForAI(options = {}) {
         const page = this.ensurePage();
         return withHelperTimeout('takeScreenshotForAI', options.timeoutMs, async () => {
             await page.mouse.move(0, 0);
-            // Normalize the screenshot to viewport CSS pixels via browser canvas.
+            await ensureCaptureHideStyles(page);
+            // Normalize the screenshot to viewport CSS pixels via sharp (Node-side).
+            // This avoids the expensive round-trip of sending a full-res PNG base64
+            // to the browser via CDP for canvas resize.
             const viewport = page.viewportSize();
+            const fullBuf = await page.screenshot({ type: 'png', fullPage: false });
             if (viewport) {
-                const fullBuf = await page.screenshot({ type: 'png', fullPage: false });
-                const resized = await page.evaluate(async ({ pngBase64, w, h }) => {
-                    const img = new Image();
-                    img.src = `data:image/png;base64,${pngBase64}`;
-                    await new Promise((resolve) => { img.onload = () => resolve(); });
-                    const canvas = document.createElement('canvas');
-                    canvas.width = w;
-                    canvas.height = h;
-                    const ctx = canvas.getContext('2d');
-                    ctx.drawImage(img, 0, 0, w, h);
-                    // Return as JPEG base64 (strip the data:image/jpeg;base64, prefix)
-                    return canvas.toDataURL('image/jpeg', 0.70).split(',')[1];
-                }, { pngBase64: fullBuf.toString('base64'), w: viewport.width, h: viewport.height });
-                return Buffer.from(resized, 'base64');
-            }
-            return Buffer.from(await page.screenshot({ type: 'jpeg', quality: 70, fullPage: false }));
+                return await sharp(fullBuf)
+                    .resize(viewport.width, viewport.height, { fit: 'fill' })
+                    .jpeg({ quality: 70 })
+                    .toBuffer();
+            }
+            return await sharp(fullBuf).jpeg({ quality: 70 }).toBuffer();
         });
     }
     async getAccessibilityTree(options = {}) {
@@ -852,6 +938,7 @@ export class Browser {
                 tag: el.tag,
                 text: el.text,
                 ariaLabel: el.ariaLabel,
+                ariaHasPopup: el.ariaHasPopup,
                 role: el.role,
                 inputType: el.inputType,
             });
@@ -963,12 +1050,680 @@ export class Browser {
             return ''; // Fallback: empty DOM, accessibility tree still available
         }
     }
+    mapRawNodeType(raw) {
+        const tag = raw.tagName.toLowerCase();
+        const role = (raw.role || raw.attributes.role || '').toLowerCase();
+        const inputType = (raw.attributes.type || '').toLowerCase();
+        if (tag === 'a' || role === 'link')
+            return 'link';
+        if (tag === 'button' || role === 'button')
+            return 'button';
+        if (role === 'menuitem')
+            return 'button';
+        if (role === 'menuitemcheckbox')
+            return 'checkbox';
+        if (role === 'menuitemradio')
+            return 'radio';
+        if (role === 'option')
+            return 'button';
+        if (tag === 'input') {
+            if (inputType === 'checkbox')
+                return 'checkbox';
+            if (inputType === 'radio')
+                return 'radio';
+            if (inputType === 'range')
+                return 'slider';
+            return 'input';
+        }
+        if (tag === 'select' || role === 'combobox' || role === 'listbox')
+            return 'select';
+        if (tag === 'textarea')
+            return 'input';
+        if (role === 'switch')
+            return 'toggle';
+        if (role === 'tab')
+            return 'tab';
+        if (tag === 'nav' || role === 'navigation')
+            return 'nav';
+        if (tag === 'form' || role === 'form')
+            return 'form';
+        if (tag === 'img' || role === 'img')
+            return 'image';
+        if (tag === 'svg')
+            return raw.interactive || raw.label ? 'icon' : 'container';
+        if (/^h[1-6]$/.test(tag) || role === 'heading')
+            return 'heading';
+        if (tag === 'ul' || tag === 'ol' || role === 'list')
+            return 'list';
+        if (tag === 'table' || role === 'table' || role === 'grid')
+            return 'table';
+        if (tag === 'video')
+            return 'video';
+        if (tag === 'audio')
+            return 'audio';
+        if (tag === 'iframe')
+            return 'iframe';
+        if (tag === '#text')
+            return 'text';
+        return 'container';
+    }
+    convertRawNode(raw) {
+        return {
+            id: '',
+            type: this.mapRawNodeType(raw),
+            label: raw.label,
+            value: raw.value,
+            bounds: raw.bounds,
+            visible: raw.visible,
+            interactive: raw.interactive,
+            state: { ...raw.state },
+            style: { ...raw.style },
+            semantic: {
+                confidence: 'medium',
+                traits: [],
+            },
+            scroll: raw.scroll ? { ...raw.scroll } : undefined,
+            attributes: { ...raw.attributes, tagName: raw.tagName, ...(raw.role ? { role: raw.role } : {}) },
+            children: raw.children.map((child) => this.convertRawNode(child)),
+            sourceRef: raw.sourceRef,
+        };
+    }
+    shouldCollapseNode(node) {
+        const ownText = (node.attributes.__ownText || '').trim();
+        const semanticTag = node.attributes.__semanticTag === 'true';
+        const hasBackground = node.attributes.__hasBackground === 'true';
+        const hasBorder = node.attributes.__hasBorder === 'true';
+        const hasShadow = node.attributes.__hasShadow === 'true';
+        return !ownText
+            && !node.interactive
+            && !semanticTag
+            && node.children.length === 1
+            && !hasBackground
+            && !hasBorder
+            && !hasShadow
+            && !node.scroll;
+    }
+    collapseNode(node) {
+        const collapsedChildren = node.children.flatMap((child) => this.collapseNode(child));
+        const nextNode = { ...node, children: collapsedChildren };
+        if (this.shouldCollapseNode(nextNode)) {
+            return collapsedChildren;
+        }
+        return [nextNode];
+    }
+    collectFingerprintEntries(node, currentHint, entries) {
+        const tagName = node.attributes.tagName || node.type.toUpperCase();
+        const input = {
+            tagName,
+            role: node.attributes.role || '',
+            textContent: node.label,
+            ariaLabel: node.attributes['aria-label'] || '',
+            htmlId: node.attributes.id || '',
+            name: node.attributes.name || '',
+            inputType: node.attributes.type || '',
+            href: node.attributes.href || '',
+            placeholder: node.attributes.placeholder || '',
+            bounds: node.bounds,
+        };
+        entries.push({ node, parentHint: currentHint || `${node.bounds.x},${node.bounds.y}`, input });
+        const nextHint = (node.attributes.__semanticTag === 'true'
+            || node.interactive
+            || node.type !== 'container'
+            || node.label)
+            ? `${tagName}|${node.label.slice(0, 30)}`
+            : currentHint;
+        for (const child of node.children) {
+            this.collectFingerprintEntries(child, nextHint, entries);
+        }
+    }
+    assignFingerprints(root) {
+        const entries = [];
+        this.collectFingerprintEntries(root, '', entries);
+        const baseCounts = new Map();
+        for (const entry of entries) {
+            const base = fingerprintAKNode(entry.input);
+            baseCounts.set(base, (baseCounts.get(base) ?? 0) + 1);
+        }
+        const usedIds = new Map();
+        for (const entry of entries) {
+            const base = fingerprintAKNode(entry.input);
+            let id = base;
+            if ((baseCounts.get(base) ?? 0) > 1) {
+                id = disambiguateFingerprint(entry.input, entry.parentHint);
+            }
+            if (usedIds.has(id)) {
+                id = disambiguateFingerprint(entry.input, `${entry.parentHint}|${entry.input.bounds.x},${entry.input.bounds.y}`);
+            }
+            usedIds.set(id, 1);
+            entry.node.id = id;
+        }
+    }
+    applyTraitAnnotations(node, page) {
+        const traits = new Set(node.semantic.traits);
+        const position = (node.attributes.__position || '').toLowerCase();
+        const zIndex = Number(node.attributes.__zIndex || '0');
+        const coveragePercent = Number(node.attributes.__coveragePercent || '0');
+        if (position === 'fixed')
+            traits.add('fixed');
+        if (position === 'sticky')
+            traits.add('sticky');
+        if ((position === 'fixed' || position === 'absolute') && zIndex > 10)
+            traits.add('floating');
+        if (coveragePercent > 50 && ((node.style.opacity ?? 1) < 1 || !!node.style.bgColor))
+            traits.add('overlay');
+        if (node.bounds.y + node.bounds.h < page.viewport.height)
+            traits.add('above-fold');
+        if (node.bounds.y > page.viewport.height)
+            traits.add('below-fold');
+        if (node.bounds.w >= page.viewport.width * 0.9)
+            traits.add('full-width');
+        if (node.scroll && (node.scroll.overflowX || node.scroll.overflowY))
+            traits.add('scrollable');
+        node.semantic.traits = [...traits];
+        for (const child of node.children) {
+            this.applyTraitAnnotations(child, page);
+        }
+    }
+    findFirstPattern(root, pattern) {
+        const nodes = [root];
+        while (nodes.length > 0) {
+            const current = nodes.shift();
+            if (!current)
+                continue;
+            if (current.semantic.pattern === pattern)
+                return current;
+            nodes.unshift(...current.children);
+        }
+        return null;
+    }
+    flattenNodes(root) {
+        const nodes = [];
+        const stack = [root];
+        while (stack.length > 0) {
+            const current = stack.pop();
+            if (!current)
+                continue;
+            nodes.push(current);
+            stack.push(...current.children);
+        }
+        return nodes;
+    }
+    annotateSemanticPatterns(tree) {
+        const nodes = this.flattenNodes(tree.root);
+        const lowerLabel = (node) => `${node.label} ${node.value || ''}`.toLowerCase();
+        const hasButtons = (node) => this.flattenNodes(node).filter((child) => child.type === 'button' || child.type === 'link').length;
+        const hasLinks = (node) => this.flattenNodes(node).some((child) => child.type === 'link');
+        const hasHeading = (node) => this.flattenNodes(node).some((child) => child.type === 'heading');
+        for (const node of nodes) {
+            const text = lowerLabel(node);
+            const position = (node.attributes.__position || '').toLowerCase();
+            const zIndex = Number(node.attributes.__zIndex || '0');
+            const centered = Math.abs((node.bounds.x + node.bounds.w / 2) - tree.page.viewport.width / 2) <= Math.max(120, tree.page.viewport.width * 0.18);
+            const nearTopOrBottom = node.bounds.y < 120 || node.bounds.y + node.bounds.h > tree.page.viewport.height - 140;
+            if (!node.semantic.pattern) {
+                if ((position === 'fixed' || position === 'sticky') && nearTopOrBottom && hasButtons(node) >= 1 && hasButtons(node) <= 3 && zIndex > 1000) {
+                    node.semantic.pattern = 'cookie-banner';
+                    node.semantic.confidence = /\b(cookie|consent|privacy|rgpd|gdpr)\b/i.test(text) ? 'high' : 'medium';
+                }
+                else if (node.attributes.role === 'dialog'
+                    || node.attributes['aria-modal'] === 'true'
+                    || (centered && zIndex > 100 && (position === 'fixed' || position === 'absolute'))) {
+                    node.semantic.pattern = 'modal';
+                    node.semantic.confidence = node.attributes.role === 'dialog' || node.attributes['aria-modal'] === 'true' ? 'high' : 'medium';
+                }
+                else if (node.attributes.role === 'menu'
+                    || node.attributes.role === 'listbox'
+                    || ((position === 'absolute' || position === 'fixed')
+                        && zIndex > 20
+                        && node.bounds.h < tree.page.viewport.height * 0.7
+                        && node.bounds.w < tree.page.viewport.width * 0.7
+                        && this.flattenNodes(node).some((child) => child.interactive
+                            && child.id !== node.id
+                            && ['button', 'link', 'checkbox', 'radio', 'select', 'toggle', 'tab'].includes(child.type)))) {
+                    node.semantic.pattern = 'dropdown';
+                    node.semantic.confidence =
+                        node.attributes.role === 'menu' || node.attributes.role === 'listbox'
+                            ? 'high'
+                            : 'medium';
+                }
+                else if ((node.type === 'nav' || node.attributes.role === 'navigation') && node.bounds.y < 100 && hasLinks(node)) {
+                    node.semantic.pattern = 'navbar';
+                    node.semantic.confidence = (position === 'sticky' || position === 'fixed') ? 'high' : 'medium';
+                }
+                else if ((node.attributes.tagName || '').toLowerCase() === 'footer' && hasLinks(node)) {
+                    node.semantic.pattern = 'footer';
+                    node.semantic.confidence = 'high';
+                }
+                else if (node.type === 'form') {
+                    node.semantic.pattern = 'form';
+                    node.semantic.confidence = 'high';
+                }
+                else if (node.bounds.h > 300 && hasHeading(node) && hasButtons(node) > 0 && node.bounds.y < tree.page.viewport.height) {
+                    node.semantic.pattern = 'hero';
+                    node.semantic.confidence = 'medium';
+                }
+            }
+        }
+        for (const node of nodes) {
+            if (node.semantic.pattern)
+                continue;
+            const buttonChildren = node.children.filter((child) => child.type === 'button' || child.type === 'link');
+            if (buttonChildren.length >= 1 && buttonChildren.length <= 3) {
+                const sorted = [...buttonChildren].sort((a, b) => a.bounds.x - b.bounds.x);
+                const closeEnough = sorted.every((child, index) => index === 0 || Math.abs(child.bounds.x - sorted[index - 1].bounds.x) < 220);
+                if (closeEnough) {
+                    node.semantic.pattern = 'cta-group';
+                    node.semantic.confidence = 'medium';
+                }
+            }
+        }
+        const navbar = this.findFirstPattern(tree.root, 'navbar');
+        if (navbar) {
+            const heroCandidate = tree.root.children.find((child) => child.id !== navbar.id
+                && child.type === 'container'
+                && child.bounds.y >= navbar.bounds.y + navbar.bounds.h
+                && child.bounds.h > 300
+                && this.flattenNodes(child).some((descendant) => descendant.type === 'heading'));
+            if (heroCandidate && !heroCandidate.semantic.pattern) {
+                heroCandidate.semantic.pattern = 'hero';
+                heroCandidate.semantic.confidence = 'medium';
+            }
+        }
+    }
+    computeOverlays(tree) {
+        return this.flattenNodes(tree.root)
+            .filter((node) => {
+            const zIndex = Number(node.attributes.__zIndex || '0');
+            const position = (node.attributes.__position || '').toLowerCase();
+            return zIndex > 100 && (position === 'fixed' || position === 'absolute' || node.semantic.traits.includes('overlay'));
+        })
+            .map((node) => ({
+            nodeId: node.id,
+            zIndex: Number(node.attributes.__zIndex || '0'),
+            coveragePercent: Number(node.attributes.__coveragePercent || '0'),
+            blocksInteraction: node.attributes.__blocksInteraction === 'true',
+        }))
+            .sort((a, b) => b.zIndex - a.zIndex);
+    }
+    normalizeRawSnapshot(snapshot) {
+        const rootNode = this.convertRawNode(snapshot.root);
+        const collapsedRootChildren = rootNode.children.flatMap((child) => this.collapseNode(child));
+        const normalizedRoot = {
+            ...rootNode,
+            children: collapsedRootChildren,
+        };
+        this.assignFingerprints(normalizedRoot);
+        this.applyTraitAnnotations(normalizedRoot, snapshot.page);
+        const tree = {
+            root: normalizedRoot,
+            page: snapshot.page,
+            overlays: [],
+        };
+        this.annotateSemanticPatterns(tree);
+        tree.overlays = this.computeOverlays(tree);
+        this.akNodeIndex = buildAKNodeRuntimeIndex(tree);
+        return tree;
+    }
+    async getAKTree() {
+        const page = this.ensurePage();
+        const snapshot = await page.evaluate(() => {
+            const SKIP_TAGS = new Set(['STYLE', 'SCRIPT', 'LINK', 'META', 'NOSCRIPT', 'SOURCE', 'PATH', 'G', 'DEFS', 'CLIPPATH']);
+            const SEMANTIC_TAGS = new Set(['NAV', 'MAIN', 'HEADER', 'FOOTER', 'SECTION', 'ARTICLE', 'ASIDE', 'FORM']);
+            const TEST_ID_ATTRS = ['data-testid', 'data-test', 'data-qa', 'data-cy'];
+            const quoteForSelector = (value) => value
+                .replace(/\\/g, '\\\\')
+                .replace(/"/g, '\\"');
+            const quoteForHasText = (value) => value
+                .replace(/\\/g, '\\\\')
+                .replace(/"/g, '\\"');
+            const cssEscape = (value) => {
+                if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
+                    return CSS.escape(value);
+                }
+                return value.replace(/(^-?\d)|[^a-zA-Z0-9_-]/g, (match) => `\\${match}`);
+            };
+            const hasVisualChrome = (style) => {
+                const background = style.backgroundColor !== 'rgba(0, 0, 0, 0)' && style.backgroundColor !== 'transparent';
+                const border = ['Top', 'Right', 'Bottom', 'Left'].some((side) => {
+                    const width = style.getPropertyValue(`border-${side.toLowerCase()}-width`);
+                    const color = style.getPropertyValue(`border-${side.toLowerCase()}-color`);
+                    return parseFloat(width || '0') > 0 && color !== 'transparent' && color !== 'rgba(0, 0, 0, 0)';
+                });
+                const shadow = !!style.boxShadow && style.boxShadow !== 'none';
+                return { background, border, shadow };
+            };
+            const isElementInteractive = (el, style) => {
+                const tag = el.tagName.toLowerCase();
+                const role = (el.getAttribute('role') || '').toLowerCase();
+                if (['a', 'button', 'input', 'select', 'textarea', 'summary'].includes(tag))
+                    return true;
+                if (['button', 'link', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'checkbox', 'radio', 'switch', 'combobox', 'option', 'slider', 'spinbutton', 'listbox', 'treeitem', 'gridcell'].includes(role)) {
+                    return true;
+                }
+                return el.tabIndex >= 0
+                    || el.hasAttribute('onclick')
+                    || el.hasAttribute('aria-haspopup')
+                    || el.getAttribute('contenteditable') === 'true'
+                    || style.cursor === 'pointer';
+            };
+            const getOwnText = (el) => {
+                let text = '';
+                for (const child of Array.from(el.childNodes)) {
+                    if (child.nodeType === Node.TEXT_NODE) {
+                        const next = (child.textContent || '').replace(/\s+/g, ' ').trim();
+                        if (next)
+                            text += `${text ? ' ' : ''}${next}`;
+                    }
+                }
+                return text.trim();
+            };
+            const getShallowText = (el) => {
+                const own = getOwnText(el);
+                if (own)
+                    return own;
+                let text = '';
+                for (const child of Array.from(el.children)) {
+                    const next = getOwnText(child);
+                    if (next)
+                        text += `${text ? ' ' : ''}${next}`;
+                }
+                return text.trim();
+            };
+            const isCodeLikeText = (value) => {
+                const normalized = value.replace(/\s+/g, ' ').trim();
+                if (!normalized)
+                    return false;
+                if (/(=>|function\s*\(|document\.|window\.|return\s+|const\s+|let\s+|var\s+)/i.test(normalized)) {
+                    return true;
+                }
+                const punctuationCount = (normalized.match(/[{}()[\];=<>]/g) ?? []).length;
+                return punctuationCount / Math.max(normalized.length, 1) > 0.12;
+            };
+            const collectVisibleDescendantText = (el, maxLength = 160) => {
+                const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT, {
+                    acceptNode(node) {
+                        const parent = node.parentElement;
+                        if (!parent)
+                            return NodeFilter.FILTER_REJECT;
+                        if (SKIP_TAGS.has(parent.tagName))
+                            return NodeFilter.FILTER_REJECT;
+                        const style = window.getComputedStyle(parent);
+                        if (style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') === 0) {
+                            return NodeFilter.FILTER_REJECT;
+                        }
+                        return NodeFilter.FILTER_ACCEPT;
+                    },
+                });
+                let text = '';
+                while (walker.nextNode() && text.length < maxLength) {
+                    const next = (walker.currentNode.textContent || '').replace(/\s+/g, ' ').trim();
+                    if (!next || isCodeLikeText(next))
+                        continue;
+                    text += `${text ? ' ' : ''}${next}`;
+                }
+                return text.slice(0, maxLength).trim();
+            };
+            const buildStructuralSelector = (el) => {
+                const segments = [];
+                let current = el;
+                while (current && current.tagName !== 'BODY' && segments.length < 6) {
+                    const tag = current.tagName.toLowerCase();
+                    const parent = current.parentElement;
+                    if (!parent)
+                        break;
+                    const sameTagSiblings = Array.from(parent.children)
+                        .filter((sibling) => sibling.tagName === current.tagName);
+                    const index = sameTagSiblings.indexOf(current) + 1;
+                    segments.unshift(`${tag}:nth-of-type(${Math.max(1, index)})`);
+                    current = parent;
+                }
+                return segments.join(' > ') || 'body';
+            };
+            const isUniqueSelector = (selector, expected) => {
+                try {
+                    const matches = document.querySelectorAll(selector);
+                    return matches.length === 1 && matches[0] === expected;
+                }
+                catch {
+                    return false;
+                }
+            };
+            const buildSourceRef = (el, label) => {
+                // Reuse the same stable-selector strategy as getInteractiveElements():
+                // generate candidate selectors from attributes and pick the first that
+                // uniquely identifies the element in the current DOM.
+                const tag = el.tagName.toLowerCase();
+                const candidates = [];
+                const push = (selector) => {
+                    if (selector && !candidates.includes(selector))
+                        candidates.push(selector);
+                };
+                const role = el.getAttribute('role');
+                const type = el.getAttribute('type');
+                const htmlId = el.getAttribute('id');
+                if (htmlId)
+                    push(`#${cssEscape(htmlId)}`);
+                for (const attr of TEST_ID_ATTRS) {
+                    const value = el.getAttribute(attr);
+                    if (!value)
+                        continue;
+                    push(`[${attr}="${quoteForSelector(value)}"]`);
+                    push(`${tag}[${attr}="${quoteForSelector(value)}"]`);
+                }
+                const name = el.getAttribute('name');
+                if (name) {
+                    push(`${tag}[name="${quoteForSelector(name)}"]`);
+                    if (type)
+                        push(`${tag}[name="${quoteForSelector(name)}"][type="${quoteForSelector(type)}"]`);
+                }
+                const rawHref = el.getAttribute('href');
+                if (tag === 'a' && rawHref) {
+                    push(`a[href="${quoteForSelector(rawHref)}"]`);
+                }
+                for (const attr of ['aria-label', 'title', 'placeholder']) {
+                    const value = el.getAttribute(attr);
+                    if (!value)
+                        continue;
+                    push(`${tag}[${attr}="${quoteForSelector(value)}"]`);
+                    if (role)
+                        push(`${tag}[role="${quoteForSelector(role)}"][${attr}="${quoteForSelector(value)}"]`);
+                    if (type)
+                        push(`${tag}[type="${quoteForSelector(type)}"][${attr}="${quoteForSelector(value)}"]`);
+                }
+                if (type)
+                    push(`${tag}[type="${quoteForSelector(type)}"]`);
+                // Pick the first unique candidate
+                for (const selector of candidates) {
+                    if (isUniqueSelector(selector, el))
+                        return selector;
+                }
+                // Fallback: :has-text() with role if available (for disambiguation)
+                if (role && label) {
+                    const roleSelector = `[role="${quoteForSelector(role)}"]:has-text("${quoteForHasText(label.slice(0, 80))}")`;
+                    if (isUniqueSelector(roleSelector, el))
+                        return roleSelector;
+                }
+                if (label) {
+                    const textSelector = `${tag}:has-text("${quoteForHasText(label.slice(0, 80))}")`;
+                    if (isUniqueSelector(textSelector, el))
+                        return textSelector;
+                }
+                return buildStructuralSelector(el);
+            };
+            const collectScrollState = (el) => {
+                const style = getComputedStyle(el);
+                const overflowY = ['auto', 'scroll'].includes(style.overflowY) && el.scrollHeight > el.clientHeight;
+                const overflowX = ['auto', 'scroll'].includes(style.overflowX) && el.scrollWidth > el.clientWidth;
+                if (!overflowY && !overflowX)
+                    return undefined;
+                return {
+                    scrollTop: Math.round(el.scrollTop),
+                    scrollLeft: Math.round(el.scrollLeft),
+                    scrollHeight: Math.round(el.scrollHeight),
+                    scrollWidth: Math.round(el.scrollWidth),
+                    clientHeight: Math.round(el.clientHeight),
+                    clientWidth: Math.round(el.clientWidth),
+                    overflowY,
+                    overflowX,
+                };
+            };
+            const collectAttributes = (el) => {
+                const attrs = {};
+                const copyAttrs = [
+                    'id', 'name', 'type', 'href', 'placeholder', 'title', 'role', 'aria-label',
+                    'aria-controls', 'aria-expanded', 'aria-haspopup', 'aria-modal', 'data-testid',
+                    'data-test', 'data-qa', 'data-cy', 'alt',
+                ];
+                for (const attr of copyAttrs) {
+                    const value = el.getAttribute(attr);
+                    if (value)
+                        attrs[attr] = value.slice(0, 180);
+                }
+                return attrs;
+            };
+            const parseNode = (element) => {
+                if (!(element instanceof HTMLElement))
+                    return null;
+                if (SKIP_TAGS.has(element.tagName))
+                    return null;
+                const style = getComputedStyle(element);
+                const rect = element.getBoundingClientRect();
+                const visible = style.display !== 'none'
+                    && style.visibility !== 'hidden'
+                    && parseFloat(style.opacity || '1') !== 0
+                    && rect.width > 0
+                    && rect.height > 0;
+                const ownText = getOwnText(element).slice(0, 160);
+                const shallowText = getShallowText(element).slice(0, 160);
+                const sanitizedDescendantText = collectVisibleDescendantText(element, 160);
+                const fallbackLabel = (element.getAttribute('aria-label')
+                    || element.getAttribute('title')
+                    || (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement
+                        ? element.placeholder || element.value
+                        : '')
+                    || element.getAttribute('alt')
+                    || shallowText
+                    || sanitizedDescendantText).slice(0, 160);
+                const interactive = isElementInteractive(element, style);
+                const children = element.tagName === 'IFRAME'
+                    ? []
+                    : Array.from(element.children)
+                        .map((child) => parseNode(child))
+                        .filter((child) => child != null);
+                const scroll = collectScrollState(element);
+                const visualChrome = hasVisualChrome(style);
+                const zIndexRaw = Number.parseInt(style.zIndex || '0', 10);
+                const zIndex = Number.isFinite(zIndexRaw) ? zIndexRaw : 0;
+                const coveragePercent = rect.width > 0 && rect.height > 0
+                    ? (Math.min(rect.width, window.innerWidth) * Math.min(rect.height, window.innerHeight) / (window.innerWidth * window.innerHeight)) * 100
+                    : 0;
+                const attributes = collectAttributes(element);
+                attributes.__ownText = ownText;
+                attributes.__semanticTag = SEMANTIC_TAGS.has(element.tagName) ? 'true' : 'false';
+                attributes.__position = style.position || '';
+                attributes.__zIndex = String(zIndex);
+                attributes.__hasBackground = visualChrome.background ? 'true' : 'false';
+                attributes.__hasBorder = visualChrome.border ? 'true' : 'false';
+                attributes.__hasShadow = visualChrome.shadow ? 'true' : 'false';
+                attributes.__coveragePercent = String(Math.round(coveragePercent));
+                attributes.__blocksInteraction = style.pointerEvents !== 'none' ? 'true' : 'false';
+                return {
+                    tagName: element.tagName,
+                    role: element.getAttribute('role'),
+                    label: fallbackLabel,
+                    ownText,
+                    value: (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || element instanceof HTMLSelectElement)
+                        ? String(element.value || '').slice(0, 160)
+                        : undefined,
+                    bounds: {
+                        x: Math.round(rect.x + window.scrollX),
+                        y: Math.round(rect.y + window.scrollY),
+                        w: Math.round(rect.width),
+                        h: Math.round(rect.height),
+                    },
+                    visible,
+                    interactive,
+                    state: {
+                        disabled: !!(element.disabled),
+                        focused: document.activeElement === element,
+                        checked: element instanceof HTMLInputElement && ['checkbox', 'radio'].includes(element.type) ? element.checked : undefined,
+                        expanded: element.hasAttribute('aria-expanded') ? element.getAttribute('aria-expanded') === 'true' : undefined,
+                        selected: element instanceof HTMLOptionElement
+                            ? element.selected
+                            : element.getAttribute('aria-selected') === 'true'
+                                ? true
+                                : undefined,
+                    },
+                    style: {
+                        bgColor: style.backgroundColor !== 'rgba(0, 0, 0, 0)' ? style.backgroundColor : undefined,
+                        fgColor: style.color || undefined,
+                        fontSize: Number.parseFloat(style.fontSize || '0') || undefined,
+                        opacity: Number.parseFloat(style.opacity || '1'),
+                    },
+                    scroll,
+                    attributes,
+                    sourceRef: buildSourceRef(element, fallbackLabel),
+                    children,
+                };
+            };
+            const makePageScroll = () => ({
+                scrollTop: Math.round(window.scrollY),
+                scrollLeft: Math.round(window.scrollX),
+                scrollHeight: Math.round(document.documentElement.scrollHeight),
+                scrollWidth: Math.round(document.documentElement.scrollWidth),
+                clientHeight: Math.round(window.innerHeight),
+                clientWidth: Math.round(window.innerWidth),
+                overflowY: document.documentElement.scrollHeight > window.innerHeight,
+                overflowX: document.documentElement.scrollWidth > window.innerWidth,
+            });
+            return {
+                root: parseNode(document.body) ?? {
+                    tagName: 'BODY',
+                    role: null,
+                    label: '',
+                    ownText: '',
+                    bounds: {
+                        x: 0,
+                        y: 0,
+                        w: Math.round(window.innerWidth),
+                        h: Math.round(window.innerHeight),
+                    },
+                    visible: true,
+                    interactive: false,
+                    state: { disabled: false, focused: false },
+                    style: { opacity: 1 },
+                    attributes: {
+                        __semanticTag: 'true',
+                        __position: 'static',
+                        __zIndex: '0',
+                        __hasBackground: 'false',
+                        __hasBorder: 'false',
+                        __hasShadow: 'false',
+                        __coveragePercent: '100',
+                        __blocksInteraction: 'false',
+                    },
+                    sourceRef: 'body',
+                    children: [],
+                },
+                page: {
+                    url: window.location.href,
+                    title: document.title,
+                    viewport: {
+                        width: Math.round(window.innerWidth),
+                        height: Math.round(window.innerHeight),
+                    },
+                    scroll: makePageScroll(),
+                },
+            };
+        });
+        return this.normalizeRawSnapshot(snapshot);
+    }
     async getPageState(opts) {
         const page = this.ensurePage();
-        const [cleanScreenshot, accessibilityTree, interactiveElements, scrollInfo, simplifiedDOM] = await Promise.all([
+        const [cleanScreenshot, akTree, accessibilityTree, scrollInfo, simplifiedDOM] = await Promise.all([
             this.takeScreenshotForAI(),
+            this.getAKTree(),
             this.getAccessibilityTree(),
-            this.getInteractiveElements(),
             page.evaluate(() => ({
                 scrollY: Math.round(window.scrollY),
                 scrollHeight: document.documentElement.scrollHeight,
@@ -976,12 +1731,16 @@ export class Browser {
             })),
             this.getSimplifiedDOM(),
         ]);
+        const interactiveElements = deriveInteractiveElementsFromAKTree(akTree);
         const annotatedScreenshot = opts?.skipAnnotation
             ? cleanScreenshot
             : await annotateWithSetOfMarks(cleanScreenshot, interactiveElements);
+        const serializedAKTree = serializeAKTree(akTree);
         return {
             cleanScreenshot,
             screenshot: annotatedScreenshot,
+            akTree,
+            serializedAKTree,
             accessibilityTree,
             interactiveElements,
             simplifiedDOM,
@@ -991,9 +1750,9 @@ export class Browser {
     /** Lightweight page state without screenshots — skips takeScreenshotForAI + SoM annotation. */
     async getPageStateLite() {
         const page = this.ensurePage();
-        const [accessibilityTree, interactiveElements, scrollInfo, simplifiedDOM] = await Promise.all([
+        const [akTree, accessibilityTree, scrollInfo, simplifiedDOM] = await Promise.all([
+            this.getAKTree(),
             this.getAccessibilityTree(),
-            this.getInteractiveElements(),
             page.evaluate(() => ({
                 scrollY: Math.round(window.scrollY),
                 scrollHeight: document.documentElement.scrollHeight,
@@ -1001,7 +1760,15 @@ export class Browser {
             })),
             this.getSimplifiedDOM(),
         ]);
-        return { accessibilityTree, interactiveElements, simplifiedDOM, scrollInfo };
+        const interactiveElements = deriveInteractiveElementsFromAKTree(akTree);
+        return {
+            akTree,
+            serializedAKTree: serializeAKTree(akTree),
+            accessibilityTree,
+            interactiveElements,
+            simplifiedDOM,
+            scrollInfo,
+        };
     }
     async exportStorageState() {
         const context = this.ensureContext();
@@ -1221,11 +1988,23 @@ export class Browser {
                 const rect = node.getBoundingClientRect();
                 return rect.width > 0 && rect.height > 0;
             };
+            const isCodeLikeText = (value) => {
+                const normalized = value.replace(/\s+/g, ' ').trim();
+                if (!normalized)
+                    return false;
+                if (/(=>|function\s*\(|document\.|window\.|return\s+|const\s+|let\s+|var\s+)/i.test(normalized)) {
+                    return true;
+                }
+                const punctuationCount = (normalized.match(/[{}()[\];=<>]/g) ?? []).length;
+                return punctuationCount / Math.max(normalized.length, 1) > 0.12;
+            };
             const textFor = (node) => {
                 if (!(node instanceof HTMLElement))
                     return '';
                 const aria = node.getAttribute('aria-label') || node.getAttribute('title') || '';
-                const text = (node.innerText || node.textContent || '').replace(/\s+/g, ' ').trim();
+                const text = (node.innerText || '').replace(/\s+/g, ' ').trim();
+                if (isCodeLikeText(text))
+                    return aria.slice(0, 120);
                 return (text || aria).slice(0, 120);
             };
             const quoteForSelector = (value) => value
@@ -1316,7 +2095,7 @@ export class Browser {
                 if (!isVisible(parent))
                     continue;
                 const nextText = (walker.currentNode.textContent || '').replace(/\s+/g, ' ').trim();
-                if (!nextText)
+                if (!nextText || isCodeLikeText(nextText))
                     continue;
                 visibleText += `${visibleText ? ' ' : ''}${nextText}`;
             }
@@ -1606,9 +2385,36 @@ export class Browser {
             const root = document.body ?? document.documentElement;
             const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
             let textSample = '';
+            const dedupe = (values, max = 6) => {
+                const seen = new Set();
+                const result = [];
+                for (const value of values) {
+                    const normalized = (value || '').replace(/\s+/g, ' ').trim();
+                    if (!normalized || seen.has(normalized))
+                        continue;
+                    seen.add(normalized);
+                    result.push(normalized.slice(0, 120));
+                    if (result.length >= max)
+                        break;
+                }
+                return result;
+            };
+            const isCodeLikeText = (value) => {
+                const normalized = value.replace(/\s+/g, ' ').trim();
+                if (!normalized)
+                    return false;
+                if (/(=>|function\s*\(|document\.|window\.|return\s+|const\s+|let\s+|var\s+)/i.test(normalized)) {
+                    return true;
+                }
+                const punctuationCount = (normalized.match(/[{}()[\];=<>]/g) ?? []).length;
+                return punctuationCount / Math.max(normalized.length, 1) > 0.12;
+            };
             while (walker.nextNode() && textSample.length < 240) {
+                const parent = walker.currentNode.parentElement;
+                if (!parent)
+                    continue;
                 const nextText = (walker.currentNode.textContent || '').replace(/\s+/g, ' ').trim();
-                if (!nextText)
+                if (!nextText || isCodeLikeText(nextText))
                     continue;
                 textSample += `${textSample ? ' ' : ''}${nextText}`;
             }
@@ -1628,14 +2434,153 @@ export class Browser {
                 const rect = node.getBoundingClientRect();
                 return rect.width > 0 && rect.height > 0;
             };
-            const dialogCount = Array.from(document.querySelectorAll('dialog[open], [role="dialog"], [aria-modal="true"]')).filter(isVisible).length;
+            const textFor = (node) => {
+                if (!(node instanceof HTMLElement))
+                    return '';
+                const label = [
+                    node.getAttribute('aria-label'),
+                    node.getAttribute('title'),
+                    node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement
+                        ? node.placeholder || node.value
+                        : '',
+                    node.innerText,
+                ]
+                    .filter((value) => typeof value === 'string' && value.trim().length > 0)
+                    .join(' ')
+                    .replace(/\s+/g, ' ')
+                    .trim();
+                if (isCodeLikeText(label)) {
+                    return [
+                        node.getAttribute('aria-label'),
+                        node.getAttribute('title'),
+                        node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement
+                            ? node.placeholder || node.value
+                            : '',
+                    ]
+                        .filter((value) => typeof value === 'string' && value.trim().length > 0)
+                        .join(' ')
+                        .replace(/\s+/g, ' ')
+                        .trim()
+                        .slice(0, 160);
+                }
+                return label.slice(0, 160);
+            };
+            const collectVisibleTexts = (selector, max = 6) => dedupe(Array.from(document.querySelectorAll(selector))
+                .filter(isVisible)
+                .map((node) => textFor(node)), max);
+            const isLikelyModalSurface = (node) => {
+                if (!(node instanceof HTMLElement) || !isVisible(node))
+                    return false;
+                const style = window.getComputedStyle(node);
+                const rect = node.getBoundingClientRect();
+                if (rect.width < 180 || rect.height < 80)
+                    return false;
+                if (!['fixed', 'absolute', 'sticky'].includes(style.position))
+                    return false;
+                const centeredEnough = Math.abs((rect.left + rect.width / 2) - window.innerWidth / 2) <= Math.max(120, window.innerWidth * 0.18)
+                    && Math.abs((rect.top + rect.height / 2) - window.innerHeight / 2) <= Math.max(120, window.innerHeight * 0.18);
+                const coversLargeArea = rect.width >= window.innerWidth * 0.35 && rect.height >= window.innerHeight * 0.2;
+                const overlayLikeSelector = [
+                    node.getAttribute('data-testid') || '',
+                    node.getAttribute('data-test') || '',
+                    node.className || '',
+                    node.id || '',
+                ].join(' ').toLowerCase();
+                const overlayLikeName = /\b(modal|dialog|drawer|sheet|popover|popup|overlay)\b/.test(overlayLikeSelector);
+                return overlayLikeName && (centeredEnough || coversLargeArea);
+            };
+            const semanticDialogs = Array.from(document.querySelectorAll('dialog[open], [role="dialog"], [aria-modal="true"]')).filter(isVisible);
+            const heuristicDialogs = Array.from(document.querySelectorAll([
+                '[data-testid*="modal" i]',
+                '[data-testid*="dialog" i]',
+                '[data-test*="modal" i]',
+                '[data-test*="dialog" i]',
+                '[class*="modal" i]',
+                '[class*="dialog" i]',
+                '[class*="drawer" i]',
+                '[class*="popover" i]',
+                '[id*="modal" i]',
+                '[id*="dialog" i]',
+                '[data-radix-dialog-content]',
+                '[data-radix-dialog-content][data-state="open"]',
+                '[data-radix-popover-content][data-state="open"]',
+                '[data-radix-portal] [data-state="open"]',
+                '[data-reach-dialog-content]',
+                '[data-reach-dialog-overlay]',
+                '[data-headlessui-dialog-panel]',
+                '[data-headlessui-dialog-backdrop]',
+                '[data-headlessui-portal] [data-headlessui-state="open"]',
+                '[data-headlessui-state="open"]',
+            ].join(','))).filter(isLikelyModalSurface);
+            const dialogCount = new Set([...semanticDialogs, ...heuristicDialogs]).size;
             const expandedCount = Array.from(document.querySelectorAll('[aria-expanded="true"], details[open]')).filter(isVisible).length;
+            const visibleMenus = Array.from(document.querySelectorAll('[role="menu"], [role="listbox"], [data-headlessui-state="open"], [data-radix-popper-content-wrapper]')).filter(isVisible);
             const loadingSelectors = [
                 '[aria-busy="true"]',
                 '[class*="spinner"]', '[class*="loading"]', '[class*="skeleton"]',
                 '[class*="Spinner"]', '[class*="Loading"]', '[class*="Skeleton"]',
             ];
             const loadingIndicatorCount = Array.from(document.querySelectorAll(loadingSelectors.join(','))).filter(isVisible).length;
+            const headingTexts = collectVisibleTexts('main h1, main h2, [role="main"] h1, [role="main"] h2, h1, h2', 4);
+            const breadcrumbTexts = collectVisibleTexts('[aria-label*="breadcrumb" i] a, [data-testid*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] a', 4);
+            const navTexts = collectVisibleTexts('header a, header button, nav a, nav button, aside a, aside button, [role="navigation"] a, [role="navigation"] button', 8);
+            const overlayTexts = dedupe([
+                ...semanticDialogs.map((node) => textFor(node)),
+                ...heuristicDialogs.map((node) => textFor(node)),
+                ...visibleMenus.map((node) => textFor(node)),
+            ], 6);
+            const configurationTexts = collectVisibleTexts([
+                'dialog label',
+                'dialog legend',
+                'dialog textarea',
+                'dialog input',
+                '[role="dialog"] label',
+                '[role="dialog"] legend',
+                '[role="dialog"] textarea',
+                '[role="dialog"] input',
+                '[role="dialog"] button',
+                '[role="dialog"] [data-testid*="preview" i]',
+                '[role="dialog"] [data-testid*="device" i]',
+                '[role="dialog"] [data-testid*="mockup" i]',
+                'form label',
+                'form legend',
+                'textarea',
+                '[data-testid*="settings" i]',
+                '[data-testid*="config" i]',
+                '[data-testid*="preview" i]',
+                '[data-testid*="device" i]',
+                '[data-testid*="mockup" i]',
+            ].join(','), 6);
+            const primarySurface = dedupe([
+                ...headingTexts,
+                ...breadcrumbTexts,
+                textSample.slice(0, 140),
+            ], 4).join(' | ');
+            const navigationSurface = navTexts.join(' | ');
+            const overlaySurface = overlayTexts.length > 0 ? overlayTexts.join(' | ') : null;
+            const configurationSurface = configurationTexts.length > 0 ? configurationTexts.join(' | ') : null;
+            const hasBlockingOverlay = dialogCount > 0 || (expandedCount > 0 && overlayTexts.length > 0);
+            const strongSurfaceSignature = [
+                window.location.pathname,
+                document.title || '',
+                primarySurface,
+                navigationSurface,
+                configurationSurface || '',
+                overlaySurface || '',
+                String(dialogCount),
+                String(expandedCount),
+                String(loadingIndicatorCount),
+            ]
+                .join('|')
+                .slice(0, 900);
+            const surfaceSignature = [
+                window.location.pathname,
+                document.title || '',
+                String(dialogCount),
+                String(expandedCount),
+                String(loadingIndicatorCount),
+                textSample.slice(0, 400),
+            ].join('|');
             return {
                 url: window.location.href,
                 title: document.title || '',
@@ -1648,6 +2593,13 @@ export class Browser {
                 expandedCount,
                 loadingIndicatorCount,
                 textSample: textSample.slice(0, 240),
+                surfaceSignature,
+                primarySurface,
+                overlaySurface,
+                navigationSurface,
+                configurationSurface,
+                hasBlockingOverlay,
+                strongSurfaceSignature,
             };
         });
     }
@@ -1820,6 +2772,50 @@ export class Browser {
             y: Math.round((top + bottom) / 2),
         };
     }
+    async runClickHitTest(x, y, selector) {
+        const page = this.ensurePage();
+        try {
+            return await page.evaluate(({ pointX, pointY, targetSelector }) => {
+                const hitEl = document.elementFromPoint(pointX, pointY);
+                if (!hitEl)
+                    return true;
+                if (!targetSelector)
+                    return true;
+                const target = document.querySelector(targetSelector);
+                if (!target)
+                    return true;
+                // Direct containment or identity
+                if (target.contains(hitEl) || hitEl.contains(target) || hitEl === target)
+                    return true;
+                // Label↔input relationship: clicking a label that wraps or references
+                // the target input is valid — the browser forwards the click.
+                if (target instanceof HTMLInputElement) {
+                    // hitEl is inside a <label> that wraps the target input
+                    const parentLabel = hitEl.closest('label');
+                    if (parentLabel && parentLabel.contains(target))
+                        return true;
+                    // hitEl is inside a <label for="id"> that references the target input
+                    if (parentLabel?.htmlFor && parentLabel.htmlFor === target.id)
+                        return true;
+                }
+                if (hitEl instanceof HTMLLabelElement || hitEl.closest('label')) {
+                    const label = hitEl instanceof HTMLLabelElement ? hitEl : hitEl.closest('label');
+                    if (label.contains(target))
+                        return true;
+                    if (label.htmlFor && target instanceof HTMLElement && label.htmlFor === target.id)
+                        return true;
+                }
+                return false;
+            }, {
+                pointX: x,
+                pointY: y,
+                targetSelector: selector || null,
+            });
+        }
+        catch {
+            return null;
+        }
+    }
     async withResolvedLocator(element, fn) {
         const page = this.ensurePage();
         if (!element.selector)
@@ -2122,19 +3118,333 @@ export class Browser {
         const viewport = page.viewportSize();
         if (!viewport)
             throw new Error('Viewport unavailable during click');
+        // For form controls (radio, checkbox, select) and popup triggers,
+        // prefer Playwright's locator.click() which handles actionability checks,
+        // label→input forwarding, and React event dispatch more reliably than
+        // raw coordinate clicks.
+        const tag = element.tag?.toLowerCase();
+        const role = element.role?.toLowerCase();
+        const inputType = element.inputType?.toLowerCase();
+        const isFormControl = tag === 'input' || tag === 'select' || tag === 'textarea';
+        const isPopupTrigger = element.ariaHasPopup != null && element.ariaHasPopup !== 'false';
+        const isLabelLike = tag === 'label' || role === 'label';
+        if ((isFormControl || isPopupTrigger || isLabelLike) && element.selector) {
+            const clickedViaLocator = await this.activateSelectorTarget(element.selector);
+            if (clickedViaLocator)
+                return;
+            // Locator failed — fall through to coordinate click as last resort.
+        }
         // Prefer coordinate click — it's always precise after re-centering and
         // avoids ambiguous selectors when multiple elements share the same classes.
         const point = this.getVisiblePoint(measured.boundingBox, viewport);
+        // Hit-test: verify no overlay intercepts the click target.
+        // Uses elementFromPoint to check if the element at click coordinates matches
+        // the intended target (or is a descendant/ancestor of it).
+        const hitTestPassed = await this.runClickHitTest(point.x, point.y, element.selector);
+        if (hitTestPassed === false) {
+            // An overlay is intercepting the click. Try dismissing it with Escape and retry.
+            await page.keyboard.press('Escape');
+            await page.waitForTimeout(400);
+            // Re-measure after dismiss
+            const remeasured = await this.ensureElementInView(index, 32);
+            const retryPoint = this.getVisiblePoint(remeasured.boundingBox, viewport);
+            // Second hit-test after dismiss
+            const retryHitTest = await this.runClickHitTest(retryPoint.x, retryPoint.y, element.selector);
+            if (!retryHitTest) {
+                // Still intercepted after dismiss — fall back to Playwright's click with selector
+                // which handles actionability checks natively.
+                const clickedViaLocator = await this.withResolvedLocator(element, async (locator) => {
+                    await locator.click({ timeout: 5000 });
+                    return true;
+                });
+                if (!clickedViaLocator) {
+                    if (element.selector) {
+                        throw new Error(`Click target ${index} remained intercepted after dismiss and could not be re-resolved by selector ${element.selector}`);
+                    }
+                    throw new Error(`Click target ${index} remained intercepted after dismiss and has no reusable selector`);
+                }
+                return;
+            }
+            // Click with updated coordinates after dismiss
+            await page.mouse.click(retryPoint.x, retryPoint.y);
+            return;
+        }
+        if (hitTestPassed === null && element.selector) {
+            const clickedViaLocator = await this.withResolvedLocator(element, async (locator) => {
+                await locator.click({ timeout: 5000 });
+                return true;
+            });
+            if (!clickedViaLocator) {
+                throw new Error(`Hit-test failed for click target ${index} and selector ${element.selector} could not be re-resolved`);
+            }
+            return;
+        }
         await page.mouse.click(point.x, point.y);
     }
     async clickBySelector(selector, options) {
         const page = this.ensurePage();
-        await page.click(selector, { timeout: 5000, force: options?.force });
+        const t0 = Date.now();
+        try {
+            const existence = await page.evaluate((sel) => {
+                const matches = document.querySelectorAll(sel);
+                return {
+                    url: window.location.href,
+                    count: matches.length,
+                    firstVisible: matches.length > 0 ? matches[0].offsetParent !== null : null,
+                    docState: document.readyState,
+                };
+            }, selector);
+            logger.debug(`[clickBySelector] "${selector}" — DOM check: url=${existence.url}, matches=${existence.count}, firstVisible=${existence.firstVisible}, docState=${existence.docState}`);
+        }
+        catch (probeErr) {
+            logger.debug(`[clickBySelector] "${selector}" — DOM probe failed: ${probeErr instanceof Error ? probeErr.message.split('\n')[0] : String(probeErr)}`);
+        }
+        logger.debug(`[clickBySelector] "${selector}" — trying activateSelectorTarget`);
+        const clickedViaLocator = await this.activateSelectorTarget(selector, options);
+        logger.debug(`[clickBySelector] activateSelectorTarget returned ${clickedViaLocator} after ${Date.now() - t0}ms`);
+        if (clickedViaLocator)
+            return;
+        logger.debug(`[clickBySelector] falling back to page.click("${selector}", timeout=5000)`);
+        const t1 = Date.now();
+        try {
+            await page.click(selector, { timeout: 5000, force: options?.force });
+            logger.debug(`[clickBySelector] page.click succeeded after ${Date.now() - t1}ms`);
+        }
+        catch (err) {
+            logger.debug(`[clickBySelector] page.click FAILED after ${Date.now() - t1}ms — ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`);
+            throw err;
+        }
+    }
+    async resolveActivationTarget(selector) {
+        const page = this.ensurePage();
+        try {
+            return await page.locator(selector).first().evaluate((node) => {
+                const ACTIONABLE_SELECTOR = [
+                    'button',
+                    'a[href]',
+                    'input',
+                    'select',
+                    'textarea',
+                    '[role="button"]',
+                    '[role="link"]',
+                    '[role="menuitem"]',
+                    '[role="option"]',
+                    '[role="radio"]',
+                    '[role="switch"]',
+                    '[role="tab"]',
+                ].join(', ');
+                const isActionable = (candidate) => {
+                    if (!(candidate instanceof HTMLElement))
+                        return false;
+                    if (candidate instanceof HTMLLabelElement)
+                        return true;
+                    if (candidate instanceof HTMLButtonElement)
+                        return true;
+                    if (candidate instanceof HTMLAnchorElement && !!candidate.href)
+                        return true;
+                    if (candidate instanceof HTMLInputElement
+                        || candidate instanceof HTMLSelectElement
+                        || candidate instanceof HTMLTextAreaElement) {
+                        return true;
+                    }
+                    const role = (candidate.getAttribute('role') || '').toLowerCase();
+                    return ['button', 'link', 'menuitem', 'option', 'radio', 'switch', 'tab'].includes(role);
+                };
+                const findActivationNode = (start) => {
+                    let current = start;
+                    while (current) {
+                        if (current instanceof HTMLLabelElement) {
+                            return current.control instanceof HTMLElement ? current.control : current;
+                        }
+                        if (isActionable(current))
+                            return current;
+                        current = current.parentElement;
+                    }
+                    if (!(start instanceof HTMLElement))
+                        return null;
+                    const labelAncestor = start.closest('label');
+                    if (labelAncestor instanceof HTMLLabelElement) {
+                        return labelAncestor.control instanceof HTMLElement ? labelAncestor.control : labelAncestor;
+                    }
+                    const nested = start.querySelector(ACTIONABLE_SELECTOR);
+                    if (nested instanceof HTMLElement)
+                        return nested;
+                    return start;
+                };
+                const target = findActivationNode(node);
+                if (!target)
+                    return null;
+                const attrName = 'data-ak-activation-target';
+                const win = window;
+                if (typeof win.__akActivationTargetCounter === 'undefined') {
+                    win.__akActivationTargetCounter = 0;
+                }
+                let attrValue = target.getAttribute(attrName);
+                if (!attrValue) {
+                    win.__akActivationTargetCounter += 1;
+                    attrValue = String(win.__akActivationTargetCounter);
+                    target.setAttribute(attrName, attrValue);
+                }
+                return {
+                    selector: `[${attrName}="${attrValue}"]`,
+                    tag: target.tagName.toLowerCase(),
+                    role: target.getAttribute('role') || '',
+                    inputType: target instanceof HTMLInputElement ? target.type : target.getAttribute('type'),
+                };
+            });
+        }
+        catch {
+            return null;
+        }
+    }
+    async activateSelectorTarget(selector, options) {
+        const page = this.ensurePage();
+        const tResolve = Date.now();
+        const target = await this.resolveActivationTarget(selector);
+        logger.debug(`[activateSelectorTarget] resolveActivationTarget took ${Date.now() - tResolve}ms — target=${target ? `tag=${target.tag},role=${target.role}` : 'null'}`);
+        const activationSelector = target?.selector ?? selector;
+        const activationLocator = page.locator(activationSelector).first();
+        const targetTag = target?.tag.toLowerCase() ?? '';
+        const targetRole = target?.role.toLowerCase() ?? '';
+        const inputType = target?.inputType?.toLowerCase() ?? null;
+        const noWaitAfter = (targetTag === 'label'
+            || targetRole === 'radio'
+            || targetRole === 'switch'
+            || inputType === 'radio'
+            || inputType === 'checkbox');
+        try {
+            const tScroll = Date.now();
+            await activationLocator.scrollIntoViewIfNeeded({ timeout: 3000 }).catch((err) => {
+                logger.debug(`[activateSelectorTarget] scrollIntoViewIfNeeded swallowed (${Date.now() - tScroll}ms): ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`);
+            });
+            logger.debug(`[activateSelectorTarget] scrollIntoViewIfNeeded done in ${Date.now() - tScroll}ms`);
+            if (inputType === 'radio' || inputType === 'checkbox') {
+                await activationLocator.check({ timeout: 5000, force: options?.force, noWaitAfter: true });
+                return true;
+            }
+            const tClick = Date.now();
+            await activationLocator.click({ timeout: 5000, force: options?.force, noWaitAfter });
+            logger.debug(`[activateSelectorTarget] primary click succeeded in ${Date.now() - tClick}ms`);
+            return true;
+        }
+        catch (err) {
+            logger.debug(`[activateSelectorTarget] primary click failed: ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`);
+            if (!target || activationSelector === selector) {
+                return false;
+            }
+            try {
+                const fallbackLocator = page.locator(selector).first();
+                await fallbackLocator.scrollIntoViewIfNeeded({ timeout: 3000 }).catch(() => undefined);
+                const tFb = Date.now();
+                await fallbackLocator.click({ timeout: 5000, force: options?.force, noWaitAfter });
+                logger.debug(`[activateSelectorTarget] fallback click succeeded in ${Date.now() - tFb}ms`);
+                return true;
+            }
+            catch (err2) {
+                logger.debug(`[activateSelectorTarget] fallback click failed: ${err2 instanceof Error ? err2.message.split('\n')[0] : String(err2)}`);
+                return false;
+            }
+        }
     }
     async clickByCoordinates(x, y) {
         const page = this.ensurePage();
         await page.mouse.click(x, y);
     }
+    buildElementSnapshotFromAKEntry(entry) {
+        return {
+            selector: entry.sourceRef,
+            boundingBox: null,
+            pageBox: {
+                x: entry.bounds.x,
+                y: entry.bounds.y,
+                width: entry.bounds.w,
+                height: entry.bounds.h,
+            },
+            scrollPos: { x: 0, y: 0 },
+            tag: (entry.attributes.tagName || entry.type || 'div').toLowerCase(),
+            text: entry.label || entry.attributes.__ownText || '',
+            ariaLabel: entry.attributes['aria-label'] || null,
+            ariaHasPopup: entry.attributes['aria-haspopup'] || null,
+            role: entry.attributes.role || '',
+            inputType: entry.attributes.type || null,
+        };
+    }
+    async activateAKNodeEntry(entry, options) {
+        const page = this.ensurePage();
+        const snapshot = this.buildElementSnapshotFromAKEntry(entry);
+        const targetTag = snapshot.tag.toLowerCase();
+        const targetRole = snapshot.role.toLowerCase();
+        const inputType = snapshot.inputType?.toLowerCase() ?? null;
+        const noWaitAfter = (targetTag === 'label'
+            || targetRole === 'label'
+            || targetRole === 'radio'
+            || targetRole === 'switch'
+            || targetRole === 'option'
+            || targetRole.startsWith('menuitem')
+            || inputType === 'radio'
+            || inputType === 'checkbox');
+        const activated = await this.withResolvedLocator(snapshot, async (locator) => {
+            await locator.scrollIntoViewIfNeeded().catch(() => undefined);
+            if (targetRole.startsWith('menuitem')) {
+                await locator.evaluate((node) => {
+                    if (!(node instanceof HTMLElement))
+                        return false;
+                    node.click();
+                    return true;
+                });
+                return true;
+            }
+            if (inputType === 'radio' || inputType === 'checkbox') {
+                await locator.check({ timeout: 3000, force: options?.force, noWaitAfter: true });
+                return true;
+            }
+            await locator.click({ timeout: 3000, force: options?.force, noWaitAfter });
+            return true;
+        });
+        if (activated)
+            return true;
+        const clickedViaSelector = await this.activateSelectorTarget(entry.sourceRef, options);
+        if (clickedViaSelector)
+            return true;
+        try {
+            const locator = page.locator(entry.sourceRef).first();
+            await locator.scrollIntoViewIfNeeded().catch(() => undefined);
+            await locator.click({ timeout: 1500, force: options?.force, noWaitAfter });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async clickAKNodeEntryByBounds(entry) {
+        const page = this.ensurePage();
+        const viewport = page.viewportSize();
+        if (!viewport)
+            throw new Error('Viewport unavailable during AK node click');
+        const maxScroll = await page.evaluate(() => {
+            const root = document.scrollingElement || document.documentElement;
+            return {
+                x: Math.max(0, root.scrollWidth - window.innerWidth),
+                y: Math.max(0, root.scrollHeight - window.innerHeight),
+            };
+        });
+        const targetScrollX = Math.max(0, Math.min(Math.round(entry.bounds.x + entry.bounds.w / 2 - viewport.width / 2), maxScroll.x));
+        const targetScrollY = Math.max(0, Math.min(Math.round(entry.bounds.y + entry.bounds.h / 2 - viewport.height / 2), maxScroll.y));
+        await page.evaluate(({ x, y }) => window.scrollTo(x, y), { x: targetScrollX, y: targetScrollY });
+        await page.waitForTimeout(150);
+        const scroll = await page.evaluate(() => ({ x: window.scrollX, y: window.scrollY }));
+        const point = this.getVisiblePoint({
+            x: entry.bounds.x - scroll.x,
+            y: entry.bounds.y - scroll.y,
+            width: entry.bounds.w,
+            height: entry.bounds.h,
+        }, viewport);
+        const hitTestPassed = await this.runClickHitTest(point.x, point.y, entry.sourceRef);
+        if (hitTestPassed === false) {
+            throw new Error(`AK node "${entry.id}" remained intercepted during coordinate fallback`);
+        }
+        await page.mouse.click(point.x, point.y);
+    }
     async typeText(text, options) {
         const page = this.ensurePage();
         if (options?.index !== undefined) {
@@ -2208,6 +3518,57 @@ export class Browser {
             await page.evaluate(({ deltaX, deltaY }) => window.scrollBy(deltaX, deltaY), { deltaX: dx, deltaY: dy });
         }
     }
+    async centerNodeInView(centerNodeId, options = {}) {
+        const page = this.ensurePage();
+        const centerEntry = await this.resolveAKNode(centerNodeId);
+        if (!centerEntry?.sourceRef) {
+            throw new Error(`Unable to resolve center node "${centerNodeId}"`);
+        }
+        const centerLocator = page.locator(centerEntry.sourceRef).first();
+        const centerRect = await centerLocator.evaluate((el) => {
+            const rect = el.getBoundingClientRect();
+            return {
+                top: rect.top,
+                left: rect.left,
+                width: rect.width,
+                height: rect.height,
+            };
+        });
+        const bias = options.offset ?? 0;
+        if (options.containerNodeId) {
+            const containerEntry = await this.resolveAKNode(options.containerNodeId);
+            if (!containerEntry?.sourceRef) {
+                throw new Error(`Unable to resolve scroll container "${options.containerNodeId}"`);
+            }
+            const containerLocator = page.locator(containerEntry.sourceRef).first();
+            const containerRect = await containerLocator.evaluate((el) => {
+                const rect = el.getBoundingClientRect();
+                return {
+                    top: rect.top,
+                    left: rect.left,
+                    width: rect.width,
+                    height: rect.height,
+                };
+            });
+            const deltaX = Math.round((centerRect.left + centerRect.width / 2)
+                - (containerRect.left + containerRect.width / 2)
+                - bias);
+            const deltaY = Math.round((centerRect.top + centerRect.height / 2)
+                - (containerRect.top + containerRect.height / 2)
+                - bias);
+            await containerLocator.evaluate((el, deltas) => {
+                el.scrollBy(deltas.deltaX, deltas.deltaY);
+            }, { deltaX, deltaY });
+            return;
+        }
+        const viewport = page.viewportSize();
+        if (!viewport) {
+            throw new Error('Viewport unavailable during centered scroll');
+        }
+        const deltaX = Math.round((centerRect.left + centerRect.width / 2) - (viewport.width / 2) - bias);
+        const deltaY = Math.round((centerRect.top + centerRect.height / 2) - (viewport.height / 2) - bias);
+        await page.evaluate(({ dx, dy }) => window.scrollBy(dx, dy), { dx: deltaX, dy: deltaY });
+    }
     async pressKey(key) {
         const page = this.ensurePage();
         await page.keyboard.press(key);
@@ -2541,6 +3902,7 @@ export class Browser {
     /** Temporarily hide all fixed/sticky overlays (navbars, banners, FABs, etc.) */
     async hideFixedOverlays() {
         const page = this.ensurePage();
+        await ensureCaptureHideStyles(page);
         await page.evaluate(() => {
             document.querySelectorAll('*').forEach(el => {
                 const pos = getComputedStyle(el).position;
@@ -2832,6 +4194,94 @@ export class Browser {
             await this.restoreFixedOverlays();
         }
     }
+    async resolveAKNode(nodeId, options = {}) {
+        if (options.reparse === true) {
+            // Force a fresh tree parse to get updated sourceRefs
+            this.akNodeIndex.clear();
+            await this.getAKTree();
+            return this.akNodeIndex.get(nodeId) ?? null;
+        }
+        const cached = this.akNodeIndex.get(nodeId);
+        if (cached)
+            return cached;
+        if (options.reparse === false)
+            return null;
+        await this.getAKTree();
+        return this.akNodeIndex.get(nodeId) ?? null;
+    }
+    async tapNode(nodeId, options) {
+        let entry = await this.resolveAKNode(nodeId);
+        if (!entry) {
+            throw new Error(`Unable to resolve node "${nodeId}" to a live sourceRef`);
+        }
+        const tryTapEntry = async (candidate) => {
+            const activated = await this.activateAKNodeEntry(candidate, options);
+            if (activated)
+                return;
+            await this.clickAKNodeEntryByBounds(candidate);
+        };
+        try {
+            await tryTapEntry(entry);
+        }
+        catch (err) {
+            // sourceRef may be stale — re-parse the tree and retry once
+            entry = await this.resolveAKNode(nodeId, { reparse: true });
+            if (!entry)
+                throw err;
+            await tryTapEntry(entry);
+        }
+    }
+    async typeIntoNode(nodeId, text, options) {
+        // Default to clearFirst: false → uses page.click(selector) + keyboard.type(text)
+        // which types character-by-character and triggers all React onChange/onInput events.
+        // page.fill() (clearFirst: true) sets the value atomically and can miss controlled
+        // component handlers in frameworks like React/Supabase Auth.
+        const clearFirst = options?.clearFirst ?? false;
+        let entry = await this.resolveAKNode(nodeId);
+        if (!entry || !entry.sourceRef) {
+            throw new Error(`Unable to resolve node "${nodeId}" to a live sourceRef`);
+        }
+        try {
+            await this.typeText(text, {
+                selector: entry.sourceRef,
+                clearFirst,
+            });
+        }
+        catch (err) {
+            // sourceRef may be stale — re-parse the tree and retry once
+            entry = await this.resolveAKNode(nodeId, { reparse: true });
+            if (!entry || !entry.sourceRef)
+                throw err;
+            await this.typeText(text, {
+                selector: entry.sourceRef,
+                clearFirst,
+            });
+        }
+    }
+    async captureNode(nodeId) {
+        if (!nodeId) {
+            return this.takeScreenshot();
+        }
+        const entry = await this.resolveAKNode(nodeId);
+        if (!entry) {
+            throw new Error(`Unable to resolve node "${nodeId}" for capture`);
+        }
+        return this.screenshotByRegion({
+            x: entry.bounds.x,
+            y: entry.bounds.y,
+            width: entry.bounds.w,
+            height: entry.bounds.h,
+        });
+    }
+    async focusTree(query) {
+        const akTree = await this.getAKTree();
+        const result = focusAKTree(akTree, query);
+        return {
+            tree: result.tree,
+            serialized: result.serialized,
+            matches: result.matches,
+        };
+    }
     async wait(ms) {
         const page = this.ensurePage();
         await page.waitForTimeout(Math.min(ms, 5000));