agent-web-interface 4.2.0 → 4.4.0

package/dist/src/snapshot/snapshot-compiler.js

@@ -10,293 +10,14 @@
  * - Accessibility: Semantic information
  * - CSS: Computed styles (optional, for layout)
  */
-import { createExtractorContext, extractDom, extractAx, extractLayout, extractState, resolveLabel, resolveRegion, buildLocators, resolveGrouping, classifyAxRole, extractAttributes, } from './extractors/index.js';
+import { createExtractorContext, extractDom, extractAx, extractLayout, extractState, resolveLabel, resolveRegion, buildLocators, resolveGrouping, classifyAxRole, extractAttributes, LIVE_REGION_AX_ROLES, } from './extractors/index.js';
 import { detectInteractivity } from './extractors/interactivity-detector.js';
 import { getTextContent } from '../lib/text-utils.js';
-const ROOT_CONTEXT = 'root';
-const LIGHT_DOM_CONTEXT = 'light';
-/**
- * Build a context key based on iframe and shadow ancestry.
- */
-function buildContextKey(node) {
-    const frameKey = node.framePath?.length ? node.framePath.join('/') : ROOT_CONTEXT;
-    const shadowKey = node.shadowPath?.length ? node.shadowPath.join('/') : LIGHT_DOM_CONTEXT;
-    return `${frameKey}|${shadowKey}`;
-}
-/**
- * Build context-scoped ID maps to avoid cross-frame/shadow collisions.
- */
-function buildIdMapsByContext(domResult) {
-    const idMaps = new Map();
-    for (const node of domResult.nodes.values()) {
-        const id = node.attributes?.id;
-        if (!id)
-            continue;
-        const contextKey = buildContextKey(node);
-        let map = idMaps.get(contextKey);
-        if (!map) {
-            map = new Map();
-            idMaps.set(contextKey, map);
-        }
-        map.set(id, node);
-    }
-    return idMaps;
-}
-/**
- * Get the ID map scoped to a node's frame/shadow context.
- */
-function getIdMapForNode(node, idMapsByContext) {
-    if (!node)
-        return undefined;
-    return idMapsByContext.get(buildContextKey(node));
-}
-/**
- * Build adjacency maps for shadow roots and iframe content documents.
- * Single O(n) pass through all nodes.
- *
- * @param domResult - DOM extraction result
- * @returns Adjacency maps for shadow roots and content documents
- */
-function buildAdjacencyMaps(domResult) {
-    const shadowRootsByHost = new Map();
-    const contentDocsByFrame = new Map();
-    for (const [nodeId, node] of domResult.nodes) {
-        if (node.parentId === undefined)
-            continue;
-        if (node.nodeName === '#document-fragment') {
-            // This is a shadow root - add to shadow host's children
-            const existing = shadowRootsByHost.get(node.parentId) ?? [];
-            existing.push(nodeId);
-            shadowRootsByHost.set(node.parentId, existing);
-        }
-        else if (node.nodeName === '#document') {
-            // This is a content document - add to iframe's children
-            const existing = contentDocsByFrame.get(node.parentId) ?? [];
-            existing.push(nodeId);
-            contentDocsByFrame.set(node.parentId, existing);
-        }
-    }
-    return { shadowRootsByHost, contentDocsByFrame };
-}
-/**
- * Build DOM pre-order index by traversing the DOM tree.
- * Also traverses into shadow roots and iframe content documents.
- *
- * @param domResult - DOM extraction result with nodes and rootId
- * @param adjacencyMaps - Precomputed maps for shadow roots and content documents
- * @returns Map of backendNodeId -> DOM order index
- */
-function buildDomOrderIndex(domResult, adjacencyMaps) {
-    const orderIndex = new Map();
-    const shadowHostSet = new Set(domResult.shadowRoots);
-    let index = 0;
-    function traverse(nodeId) {
-        const node = domResult.nodes.get(nodeId);
-        if (!node)
-            return;
-        orderIndex.set(nodeId, index++);
-        // 1. Process light DOM children first (pre-order DFS)
-        if (node.childNodeIds) {
-            for (const childId of node.childNodeIds) {
-                traverse(childId);
-            }
-        }
-        // 2. If this node hosts a shadow root, traverse shadow content (O(1) lookup)
-        if (shadowHostSet.has(nodeId)) {
-            const shadowRoots = adjacencyMaps.shadowRootsByHost.get(nodeId) ?? [];
-            for (const shadowRootId of shadowRoots) {
-                traverse(shadowRootId);
-            }
-        }
-        // 3. If this node is an iframe, traverse content document (O(1) lookup)
-        if (node.frameId || node.nodeName.toUpperCase() === 'IFRAME') {
-            const contentDocs = adjacencyMaps.contentDocsByFrame.get(nodeId) ?? [];
-            for (const contentDocId of contentDocs) {
-                traverse(contentDocId);
-            }
-        }
-    }
-    traverse(domResult.rootId);
-    return orderIndex;
-}
-/**
- * Build heading index mapping each backendNodeId to its heading context.
- * Uses DOM order to determine the most recent preceding heading.
- * Also traverses into shadow roots and iframe content documents.
- *
- * Heading context is isolated at iframe boundaries:
- * - Heading from parent document does NOT propagate into iframe
- * - Heading from iframe does NOT propagate back to parent document
- * - Shadow DOM shares heading context with its host document
- *
- * @param domResult - DOM extraction result
- * @param axResult - AX extraction result for heading names
- * @param idMap - Map of DOM ID to RawDomNode for aria-labelledby resolution
- * @param adjacencyMaps - Precomputed maps for shadow roots and content documents
- * @returns Map of backendNodeId -> heading context string
- */
-function buildHeadingIndex(domResult, axResult, idMapsByContext, adjacencyMaps) {
-    const headingIndex = new Map();
-    const shadowHostSet = new Set(domResult.shadowRoots);
-    // Helper to check if a node is a heading and resolve its name
-    function isHeading(backendNodeId) {
-        const domNode = domResult.nodes.get(backendNodeId);
-        const axNode = axResult?.nodes.get(backendNodeId);
-        // Check AX role first
-        const scopedIdMap = domNode ? getIdMapForNode(domNode, idMapsByContext) : undefined;
-        if (axNode?.role === 'heading') {
-            // Priority: AX name -> resolveLabel -> DOM text content
-            let name = axNode.name;
-            if (!name && domNode) {
-                const labelResult = resolveLabel(domNode, axNode, scopedIdMap);
-                if (labelResult.source !== 'none') {
-                    name = labelResult.label;
-                }
-            }
-            name ??= getTextContent(backendNodeId, domResult.nodes);
-            return { isHeading: true, name };
-        }
-        // Check DOM tag (H1-H6)
-        if (domNode?.nodeName?.match(/^H[1-6]$/i)) {
-            // Priority: AX name -> resolveLabel -> DOM text content
-            let name = axNode?.name;
-            if (!name) {
-                const labelResult = resolveLabel(domNode, axNode, scopedIdMap);
-                if (labelResult.source !== 'none') {
-                    name = labelResult.label;
-                }
-            }
-            name ??= getTextContent(backendNodeId, domResult.nodes);
-            return { isHeading: true, name };
-        }
-        return { isHeading: false };
-    }
-    // Traverse DOM in pre-order, passing and returning heading context
-    function traverse(nodeId, currentHeading) {
-        const node = domResult.nodes.get(nodeId);
-        if (!node)
-            return currentHeading;
-        // Check if this node is a heading
-        const headingInfo = isHeading(nodeId);
-        if (headingInfo.isHeading && headingInfo.name) {
-            currentHeading = headingInfo.name;
-        }
-        // Record the current heading context for this node
-        if (currentHeading) {
-            headingIndex.set(nodeId, currentHeading);
-        }
-        // 1. Process light DOM children first (pre-order DFS)
-        // Heading context propagates and updates through light DOM
-        if (node.childNodeIds) {
-            for (const childId of node.childNodeIds) {
-                currentHeading = traverse(childId, currentHeading) ?? currentHeading;
-            }
-        }
-        // 2. If this node hosts a shadow root, traverse shadow content (O(1) lookup)
-        // Shadow DOM shares heading context with host document (same logical document)
-        if (shadowHostSet.has(nodeId)) {
-            const shadowRoots = adjacencyMaps.shadowRootsByHost.get(nodeId) ?? [];
-            for (const shadowRootId of shadowRoots) {
-                currentHeading = traverse(shadowRootId, currentHeading) ?? currentHeading;
-            }
-        }
-        // 3. If this node is an iframe, traverse content document (O(1) lookup)
-        // IMPORTANT: Heading context resets at iframe boundary (separate document)
-        // - Pass undefined to reset context inside iframe
-        // - Discard returned heading (iframe headings don't affect parent)
-        if (node.frameId || node.nodeName.toUpperCase() === 'IFRAME') {
-            const contentDocs = adjacencyMaps.contentDocsByFrame.get(nodeId) ?? [];
-            for (const contentDocId of contentDocs) {
-                traverse(contentDocId, undefined);
-            }
-        }
-        return currentHeading;
-    }
-    traverse(domResult.rootId, undefined);
-    return headingIndex;
-}
-/**
- * Recursively collect frame loaderIds from frame tree.
- */
-function collectFrameLoaderIds(frameTree, frameLoaderIds, _isMainFrame = true) {
-    const frame = frameTree.frame;
-    frameLoaderIds.set(frame.id, {
-        frameId: frame.id,
-        loaderId: frame.loaderId,
-        isMainFrame: !frame.parentId,
-    });
-    if (frameTree.childFrames) {
-        for (const child of frameTree.childFrames) {
-            collectFrameLoaderIds(child, frameLoaderIds, false);
-        }
-    }
-}
-/**
- * Default compile options
- */
-const DEFAULT_OPTIONS = {
-    include_hidden: false,
-    max_nodes: 2000,
-    timeout: 30000,
-    redact_sensitive: true,
-    include_values: true, // Enable value extraction with password redaction
-    includeReadable: true,
-    includeLayout: true,
-};
-/**
- * Map AX role to NodeKind.
- */
-function mapRoleToKind(role) {
-    if (!role)
-        return undefined;
-    const normalized = role.toLowerCase();
-    const kindMap = {
-        // Interactive
-        button: 'button',
-        link: 'link',
-        textbox: 'input',
-        searchbox: 'input',
-        combobox: 'combobox',
-        listbox: 'select',
-        checkbox: 'checkbox',
-        radio: 'radio',
-        switch: 'switch',
-        slider: 'slider',
-        spinbutton: 'slider',
-        tab: 'tab',
-        menuitem: 'menuitem',
-        menuitemcheckbox: 'menuitem',
-        menuitemradio: 'menuitem',
-        option: 'menuitem',
-        // Readable
-        heading: 'heading',
-        paragraph: 'paragraph',
-        text: 'text',
-        statictext: 'text',
-        list: 'list',
-        listitem: 'listitem',
-        tree: 'list',
-        treeitem: 'listitem',
-        image: 'image',
-        img: 'image',
-        figure: 'image',
-        table: 'table',
-        grid: 'table',
-        treegrid: 'table',
-        // Structural
-        form: 'form',
-        dialog: 'dialog',
-        alertdialog: 'dialog',
-        navigation: 'navigation',
-        region: 'section',
-        article: 'section',
-        main: 'section',
-        banner: 'section',
-        complementary: 'section',
-        contentinfo: 'section',
-    };
-    return kindMap[normalized];
-}
+import { buildIdMapsByContext, getIdMapForNode, buildAdjacencyMaps, buildDomOrderIndex, collectFrameLoaderIds, } from './frame-context.js';
+import { buildHeadingIndex } from './heading-index.js';
+import { synthesizeOptionNodes, synthesizeCanvasNodes, promoteToastNodes, } from './node-synthesizer.js';
+import { filterNoiseNodes, sliceWithOverlayPriority } from './node-filter.js';
+import { mapRoleToKind, getKindFromTag, DEFAULT_OPTIONS, } from './kind-mapping.js';
 /**
  * SnapshotCompiler class
  *
@@ -426,7 +147,10 @@ export class SnapshotCompiler {
                 const isReadable = classification === 'readable' && this.options.includeReadable;
                 const isEssentialStructural = classification === 'structural' &&
                     essentialStructuralRoles.has(axNode.role?.toLowerCase() ?? '');
-                if (isInteractive || isReadable || isEssentialStructural) {
+                // Live region roles (alert, status, log, tooltip, progressbar, timer)
+                // are always included — they carry critical action feedback
+                const isLiveRegion = classification === 'live';
+                if (isInteractive || isReadable || isEssentialStructural || isLiveRegion) {
                     const domNode = domResult?.nodes.get(backendNodeId);
                     nodesToProcess.push({
                         backendNodeId,
@@ -440,7 +164,7 @@ export class SnapshotCompiler {
             // Fallback: Use DOM-only for interactive tags and essential structural elements
             for (const [backendNodeId, domNode] of domResult.nodes) {
                 const tagName = domNode.nodeName.toUpperCase();
-                if (['BUTTON', 'A', 'INPUT', 'SELECT', 'TEXTAREA', 'FORM', 'DIALOG'].includes(tagName)) {
+                if (['BUTTON', 'A', 'INPUT', 'SELECT', 'TEXTAREA', 'FORM', 'DIALOG', 'CANVAS'].includes(tagName)) {
                     nodesToProcess.push({
                         backendNodeId,
                         domNode,
@@ -448,65 +172,11 @@ export class SnapshotCompiler {
                 }
             }
         }
-        // Phase 2.1: Synthesize option nodes from <select> children.
-        // Chrome's AX tree often marks <option> nodes as ignored when the select
-        // is collapsed, and their bounding boxes are zero (OS-rendered).
-        // We inject them from the DOM so AI agents can discover available options.
+        // Phase 2.1-2.3: Synthesize missing nodes from DOM
         if (domResult) {
-            const alreadyInSet = new Set(nodesToProcess.map((n) => n.backendNodeId));
-            for (const nodeData of [...nodesToProcess]) {
-                const domNode = nodeData.domNode;
-                if (domNode?.nodeName.toUpperCase() !== 'SELECT')
-                    continue;
-                const collectOptions = (parentId) => {
-                    const parent = domResult.nodes.get(parentId);
-                    if (!parent?.childNodeIds)
-                        return;
-                    for (const childId of parent.childNodeIds) {
-                        const child = domResult.nodes.get(childId);
-                        if (!child)
-                            continue;
-                        const childTag = child.nodeName.toUpperCase();
-                        if (childTag === 'OPTGROUP') {
-                            // Recurse into optgroup to find nested options
-                            collectOptions(childId);
-                        }
-                        else if (childTag === 'OPTION' && !alreadyInSet.has(childId)) {
-                            // Extract text content from option's child text nodes
-                            const optionText = getTextContent(childId, domResult.nodes);
-                            // Build synthetic AX node so label resolution and state extraction work
-                            const syntheticAx = {
-                                nodeId: `synthetic-opt-${childId}`,
-                                backendDOMNodeId: childId,
-                                role: 'option',
-                                name: optionText ?? '',
-                                properties: [],
-                            };
-                            // Transfer selected attribute to AX property
-                            if (child.attributes?.selected !== undefined) {
-                                syntheticAx.properties.push({
-                                    name: 'selected',
-                                    value: { type: 'boolean', value: true },
-                                });
-                            }
-                            // Transfer disabled attribute to AX property
-                            if (child.attributes?.disabled !== undefined) {
-                                syntheticAx.properties.push({
-                                    name: 'disabled',
-                                    value: { type: 'boolean', value: true },
-                                });
-                            }
-                            nodesToProcess.push({
-                                backendNodeId: childId,
-                                domNode: child,
-                                axNode: syntheticAx,
-                            });
-                            alreadyInSet.add(childId);
-                        }
-                    }
-                };
-                collectOptions(domNode.backendNodeId);
-            }
+            synthesizeOptionNodes(nodesToProcess, domResult);
+            synthesizeCanvasNodes(nodesToProcess, domResult);
+            promoteToastNodes(nodesToProcess, domResult);
         }
         // Sort by DOM order if available (before max_nodes slicing)
         if (domOrderAvailable && domOrderIndex) {
@@ -540,6 +210,7 @@ export class SnapshotCompiler {
             'slider',
             'tab',
             'menuitem',
+            'canvas',
         ]);
         // Collect non-interactive nodes already in nodesToProcess (Case A)
         for (const nodeData of nodesToProcess) {
@@ -651,7 +322,7 @@ export class SnapshotCompiler {
             }
         }
         // Phase 4.5: Filter noise nodes (empty containers, duplicate text)
-        const nodes = this.filterNoiseNodes(transformedNodes, domResult?.nodes ?? new Map(), axResult?.nodes ?? new Map());
+        const nodes = filterNoiseNodes(transformedNodes, domResult?.nodes ?? new Map(), axResult?.nodes ?? new Map());
         // Phase 5: Build BaseSnapshot
         const duration = Date.now() - startTime;
         const interactiveCount = nodes.filter((n) => [
@@ -667,6 +338,7 @@ export class SnapshotCompiler {
             'slider',
             'tab',
             'menuitem',
+            'canvas',
         ].includes(n.kind) || n.implicitly_interactive).length;
         const meta = {
             node_count: nodes.length,
@@ -722,14 +394,22 @@ export class SnapshotCompiler {
             kind = mapRoleToKind(axNode.role) ?? 'generic';
         }
         else if (domNode) {
-            const tagKind = this.getKindFromTag(domNode.nodeName);
+            const tagKind = getKindFromTag(domNode.nodeName);
             if (tagKind)
                 kind = tagKind;
         }
         // Resolve label
         const scopedIdMap = getIdMapForNode(domNode, idMapsByContext);
         const labelResult = resolveLabel(domNode, axNode, scopedIdMap);
-        const label = labelResult.label;
+        let label = labelResult.label;
+        // Fallback for live region containers (alert, status, etc.) with empty labels:
+        // Their text typically lives in child StaticText/text nodes, not in the AX name.
+        if (!label &&
+            axNode?.role &&
+            LIVE_REGION_AX_ROLES.has(axNode.role.toLowerCase()) &&
+            domTree.size > 0) {
+            label = getTextContent(backendNodeId, domTree, 3) ?? '';
+        }
         // Resolve region (pass axTree for ancestor AX role lookup)
         const region = resolveRegion(domNode, axNode, domTree, axTree);
         // Resolve grouping (for group_id and group_path only)
@@ -797,178 +477,6 @@ export class SnapshotCompiler {
         }
         return node;
     }
-    /**
-     * Get NodeKind from HTML tag name.
-     */
-    getKindFromTag(tagName) {
-        const tag = tagName.toUpperCase();
-        const tagMap = {
-            BUTTON: 'button',
-            A: 'link',
-            INPUT: 'input',
-            TEXTAREA: 'textarea',
-            SELECT: 'select',
-            H1: 'heading',
-            H2: 'heading',
-            H3: 'heading',
-            H4: 'heading',
-            H5: 'heading',
-            H6: 'heading',
-            P: 'paragraph',
-            IMG: 'image',
-            TABLE: 'table',
-            UL: 'list',
-            OL: 'list',
-            LI: 'listitem',
-            FORM: 'form',
-            DIALOG: 'dialog',
-            NAV: 'navigation',
-            OPTION: 'menuitem',
-        };
-        return tagMap[tag];
-    }
-    /**
-     * Filter out noise nodes to reduce snapshot size.
-     *
-     * Filters:
-     * 1. Empty list/listitem containers with no semantic name AND no interactive descendants
-     * 2. StaticText/text nodes that mirror their parent's label exactly
-     */
-    filterNoiseNodes(nodes, domTree, axTree) {
-        // Build set of interactive node backend IDs for descendant checking
-        const interactiveKinds = new Set([
-            'button',
-            'link',
-            'input',
-            'textarea',
-            'select',
-            'combobox',
-            'checkbox',
-            'radio',
-            'switch',
-            'slider',
-            'tab',
-            'menuitem',
-        ]);
-        const interactiveBackendIds = new Set(nodes.filter((n) => interactiveKinds.has(n.kind)).map((n) => n.backend_node_id));
-        // Build parent-child relationship from DOM tree
-        const childToParent = new Map();
-        for (const [nodeId, domNode] of domTree) {
-            if (domNode.parentId !== undefined) {
-                childToParent.set(nodeId, domNode.parentId);
-            }
-        }
-        // Check if a node has any interactive descendants in the DOM tree
-        const hasInteractiveDescendant = (nodeId) => {
-            const domNode = domTree.get(nodeId);
-            if (!domNode)
-                return false;
-            // Check direct children
-            if (domNode.childNodeIds) {
-                for (const childId of domNode.childNodeIds) {
-                    if (interactiveBackendIds.has(childId)) {
-                        return true;
-                    }
-                    if (hasInteractiveDescendant(childId)) {
-                        return true;
-                    }
-                }
-            }
-            return false;
-        };
-        // Get label of parent node in the node list
-        const getParentLabel = (nodeId) => {
-            const parentId = childToParent.get(nodeId);
-            if (parentId === undefined)
-                return undefined;
-            // Look up parent in our node list
-            const parentNode = nodes.find((n) => n.backend_node_id === parentId);
-            if (parentNode) {
-                return parentNode.label;
-            }
-            // Parent might be further up - check AX tree for parent's name
-            const parentAx = axTree.get(parentId);
-            return parentAx?.name;
-        };
-        // Container kinds that can be noisy when empty
-        const containerKinds = new Set(['list', 'listitem']);
-        // Text kinds that can duplicate parent labels
-        const textKinds = new Set(['text']);
-        return nodes.filter((node) => {
-            // Rule 1: Filter empty container nodes without interactive descendants
-            if (containerKinds.has(node.kind)) {
-                const hasSemanticName = node.label && node.label.trim().length > 0;
-                if (!hasSemanticName) {
-                    const hasInteractive = hasInteractiveDescendant(node.backend_node_id);
-                    if (!hasInteractive) {
-                        return false; // Filter out empty container without interactive content
-                    }
-                }
-            }
-            // Rule 2: Filter text nodes that mirror parent's label
-            if (textKinds.has(node.kind)) {
-                const parentLabel = getParentLabel(node.backend_node_id);
-                if (parentLabel && node.label) {
-                    // Normalize and compare
-                    const normalizedParent = parentLabel.trim().toLowerCase();
-                    const normalizedNode = node.label.trim().toLowerCase();
-                    if (normalizedNode === normalizedParent) {
-                        return false; // Filter out duplicate text
-                    }
-                }
-            }
-            return true; // Keep the node
-        });
-    }
-}
-/**
- * Slice nodes to max_nodes budget while preserving high z-index overlay content.
- *
- * Portal-rendered content (dropdowns, popovers, modals) appears at the end of
- * DOM order. On heavy pages, a naive slice truncates it.
- *
- * Strategy:
- * 1. Partition nodes into overlay (z-index > threshold) and main
- * 2. Take all overlay nodes (up to 30% of budget)
- * 3. Fill remaining budget with main nodes (DOM order)
- * 4. Re-sort by original DOM order
- */
-export function sliceWithOverlayPriority(nodes, maxNodes) {
-    if (nodes.length <= maxNodes) {
-        return nodes;
-    }
-    const OVERLAY_Z_THRESHOLD = 100;
-    const MAX_OVERLAY_RATIO = 0.3;
-    const overlayNodes = [];
-    const mainNodes = [];
-    for (const node of nodes) {
-        const zIndex = node.layout?.zIndex;
-        if (zIndex !== undefined && zIndex > OVERLAY_Z_THRESHOLD) {
-            overlayNodes.push(node);
-        }
-        else {
-            mainNodes.push(node);
-        }
-    }
-    // No overlay content → simple slice
-    if (overlayNodes.length === 0) {
-        return nodes.slice(0, maxNodes);
-    }
-    // Reserve budget for overlay (capped at 30% of total)
-    const maxOverlay = Math.min(overlayNodes.length, Math.floor(maxNodes * MAX_OVERLAY_RATIO));
-    const overlaySlice = overlayNodes.slice(0, maxOverlay);
-    // Fill remaining budget with main content
-    const mainBudget = maxNodes - overlaySlice.length;
-    const mainSlice = mainNodes.slice(0, mainBudget);
-    // Merge and re-sort by original DOM order
-    const merged = [...mainSlice, ...overlaySlice];
-    const indexMap = new Map(nodes.map((n, i) => [n.backendNodeId, i]));
-    merged.sort((a, b) => {
-        const ia = indexMap.get(a.backendNodeId) ?? Infinity;
-        const ib = indexMap.get(b.backendNodeId) ?? Infinity;
-        return ia - ib;
-    });
-    return merged;
 }
 /**
  * Export a compile function for simpler usage.