agent-web-interface 4.3.0 → 4.5.0

package/dist/src/snapshot/node-synthesizer.js

@@ -0,0 +1,185 @@
+/**
+ * Node Synthesizer
+ *
+ * Synthesizes nodes from DOM data that the AX tree misses or misclassifies:
+ * - Option nodes from <select> children (often marked as ignored by Chrome)
+ * - Canvas nodes (classified as generic/ignored by AX tree)
+ * - Toast library nodes (promoted to alert role for unsemantic toast libraries)
+ *
+ * @module snapshot/node-synthesizer
+ */
+import { getTextContent } from '../lib/text-utils.js';
+/**
+ * Data attributes used by unsemantic toast libraries.
+ * Keep in sync with TOAST_LIBRARY_SELECTOR in observer-script.ts.
+ */
+export const TOAST_DATA_ATTRS = ['data-sonner-toast', 'data-hot-toast'];
+/**
+ * CSS class substrings used by unsemantic toast libraries.
+ * Keep in sync with TOAST_LIBRARY_SELECTOR in observer-script.ts.
+ */
+export const TOAST_CLASS_PATTERNS = [
+    'Toastify__toast',
+    'ant-message-notice',
+    'ant-message',
+    'chakra-toast',
+];
+/**
+ * Check if a DOM node belongs to a known unsemantic toast library.
+ */
+export function isToastLibraryNode(domNode) {
+    const attrs = domNode.attributes;
+    if (!attrs)
+        return false;
+    for (const attrName of TOAST_DATA_ATTRS) {
+        if (attrs[attrName] !== undefined)
+            return true;
+    }
+    const className = attrs.class ?? attrs.className;
+    if (typeof className === 'string') {
+        for (const pattern of TOAST_CLASS_PATTERNS) {
+            if (className.includes(pattern))
+                return true;
+        }
+    }
+    return false;
+}
+/**
+ * Synthesize option nodes from <select> children.
+ *
+ * Chrome's AX tree often marks <option> nodes as ignored when the select
+ * is collapsed, and their bounding boxes are zero (OS-rendered).
+ * We inject them from the DOM so AI agents can discover available options.
+ *
+ * @param nodesToProcess - Current node list (mutated in place)
+ * @param domResult - DOM extraction result
+ */
+export function synthesizeOptionNodes(nodesToProcess, domResult) {
+    const alreadyInSet = new Set(nodesToProcess.map((n) => n.backendNodeId));
+    for (const nodeData of [...nodesToProcess]) {
+        const domNode = nodeData.domNode;
+        if (domNode?.nodeName.toUpperCase() !== 'SELECT')
+            continue;
+        const collectOptions = (parentId) => {
+            const parent = domResult.nodes.get(parentId);
+            if (!parent?.childNodeIds)
+                return;
+            for (const childId of parent.childNodeIds) {
+                const child = domResult.nodes.get(childId);
+                if (!child)
+                    continue;
+                const childTag = child.nodeName.toUpperCase();
+                if (childTag === 'OPTGROUP') {
+                    // Recurse into optgroup to find nested options
+                    collectOptions(childId);
+                }
+                else if (childTag === 'OPTION' && !alreadyInSet.has(childId)) {
+                    // Extract text content from option's child text nodes
+                    const optionText = getTextContent(childId, domResult.nodes);
+                    // Build synthetic AX node so label resolution and state extraction work
+                    const syntheticAx = {
+                        nodeId: `synthetic-opt-${childId}`,
+                        backendDOMNodeId: childId,
+                        role: 'option',
+                        name: optionText ?? '',
+                        properties: [],
+                    };
+                    // Transfer selected attribute to AX property
+                    if (child.attributes?.selected !== undefined) {
+                        syntheticAx.properties.push({
+                            name: 'selected',
+                            value: { type: 'boolean', value: true },
+                        });
+                    }
+                    // Transfer disabled attribute to AX property
+                    if (child.attributes?.disabled !== undefined) {
+                        syntheticAx.properties.push({
+                            name: 'disabled',
+                            value: { type: 'boolean', value: true },
+                        });
+                    }
+                    nodesToProcess.push({
+                        backendNodeId: childId,
+                        domNode: child,
+                        axNode: syntheticAx,
+                    });
+                    alreadyInSet.add(childId);
+                }
+            }
+        };
+        collectOptions(domNode.backendNodeId);
+    }
+}
+/**
+ * Synthesize canvas nodes from DOM.
+ *
+ * Canvas elements are classified as generic/ignored by the AX tree,
+ * so we inject them from the DOM so AI agents can discover and interact with them.
+ *
+ * @param nodesToProcess - Current node list (mutated in place)
+ * @param domResult - DOM extraction result
+ */
+export function synthesizeCanvasNodes(nodesToProcess, domResult) {
+    const alreadyInCanvas = new Set(nodesToProcess.map((n) => n.backendNodeId));
+    for (const [backendNodeId, domNode] of domResult.nodes) {
+        if (domNode.nodeName.toUpperCase() !== 'CANVAS')
+            continue;
+        if (alreadyInCanvas.has(backendNodeId))
+            continue;
+        const syntheticAx = {
+            nodeId: `synthetic-canvas-${backendNodeId}`,
+            backendDOMNodeId: backendNodeId,
+            role: 'canvas',
+            name: domNode.attributes?.['aria-label'] ?? '',
+            properties: [],
+        };
+        nodesToProcess.push({
+            backendNodeId,
+            domNode,
+            axNode: syntheticAx,
+        });
+        alreadyInCanvas.add(backendNodeId);
+    }
+}
+/**
+ * Promote unsemantic toast library nodes to alert role.
+ *
+ * Popular toast libraries (Sonner, react-hot-toast, Toastify, Ant Design, Chakra UI)
+ * don't use ARIA roles. Detect them by data attributes and CSS classes.
+ * Selectors mirror TOAST_LIBRARY_SELECTOR in observer-script.ts — keep in sync.
+ *
+ * Some toast elements (e.g., Sonner's <li data-sonner-toast>) are already in
+ * nodesToProcess via the AX tree (as listitem/generic), so we must also override
+ * their axNode role — not just add new nodes.
+ *
+ * @param nodesToProcess - Current node list (mutated in place)
+ * @param domResult - DOM extraction result
+ */
+export function promoteToastNodes(nodesToProcess, domResult) {
+    const scheduledIndex = new Map(nodesToProcess.map((n, i) => [n.backendNodeId, i]));
+    for (const [backendNodeId, domNode] of domResult.nodes) {
+        if (!isToastLibraryNode(domNode))
+            continue;
+        const syntheticAx = {
+            nodeId: `synthetic-toast-${backendNodeId}`,
+            backendDOMNodeId: backendNodeId,
+            role: 'alert',
+            name: '', // Will be resolved via label fallback (textContent)
+            properties: [],
+        };
+        const existingIdx = scheduledIndex.get(backendNodeId);
+        if (existingIdx !== undefined) {
+            // Already scheduled (e.g., Sonner <li> classified as listitem) — override role
+            nodesToProcess[existingIdx].axNode = syntheticAx;
+        }
+        else {
+            nodesToProcess.push({
+                backendNodeId,
+                domNode,
+                axNode: syntheticAx,
+            });
+            scheduledIndex.set(backendNodeId, nodesToProcess.length - 1);
+        }
+    }
+}
+//# sourceMappingURL=node-synthesizer.js.map

package/dist/src/snapshot/node-synthesizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"node-synthesizer.js","sourceRoot":"","sources":["../../../src/snapshot/node-synthesizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC;AAExE;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,iBAAiB;IACjB,oBAAoB;IACpB,aAAa;IACb,cAAc;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAmB;IACpD,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC;IACjC,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IAEzB,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;IACjD,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,SAAS,CAAC;IACjD,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,KAAK,MAAM,OAAO,IAAI,oBAAoB,EAAE,CAAC;YAC3C,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,qBAAqB,CACnC,cAA6B,EAC7B,SAA8B;IAE9B,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;IAEzE,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,cAAc,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;QACjC,IAAI,OAAO,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;YAAE,SAAS;QAE3D,MAAM,cAAc,GAAG,CAAC,QAAgB,EAAE,EAAE;YAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,EAAE,YAAY;gBAAE,OAAO;YAElC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC3C,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAE9C,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;oBAC5B,+CAA+C;oBAC/C,cAAc,CAAC,OAAO,CAAC,CAAC;gBAC1B,CAAC;qBAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/D,sDAAsD;oBACtD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;oBAE5D,wEAAwE;oBACxE,MAAM,WAAW,GAAc;wBAC7B,MAAM,EAAE,iBAAiB,OAAO,EAAE;wBAClC,gBAAgB,EAAE,OAAO;wBACzB,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,UAAU,IAAI,EAAE;wBACtB,UAAU,EAAE,EAAE;qBACf,CAAC;oBAEF,6CAA6C;oBAC7C,IAAI,KAAK,CAAC,UAAU,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;wBAC7C,WAAW,CAAC,UAAW,CAAC,IAAI,CAAC;4BAC3B,IAAI,EAAE,UAAU;4BAChB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;yBACxC,CAAC,CAAC;oBACL,CAAC;oBAED,6CAA6C;oBAC7C,IAAI,KAAK,CAAC,UAAU,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;wBAC7C,WAAW,CAAC,UAAW,CAAC,IAAI,CAAC;4BAC3B,IAAI,EAAE,UAAU;4BAChB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;yBACxC,CAAC,CAAC;oBACL,CAAC;oBAED,cAAc,CAAC,IAAI,CAAC;wBAClB,aAAa,EAAE,OAAO;wBACtB,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,WAAW;qBACpB,CAAC,CAAC;oBACH,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,cAAc,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CACnC,cAA6B,EAC7B,SAA8B;IAE9B,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACvD,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;YAAE,SAAS;QAC1D,IAAI,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC;YAAE,SAAS;QAEjD,MAAM,WAAW,GAAc;YAC7B,MAAM,EAAE,oBAAoB,aAAa,EAAE;YAC3C,gBAAgB,EAAE,aAAa;YAC/B,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE;YAC9C,UAAU,EAAE,EAAE;SACf,CAAC;QAEF,cAAc,CAAC,IAAI,CAAC;YAClB,aAAa;YACb,OAAO;YACP,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;QACH,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,iBAAiB,CAC/B,cAA6B,EAC7B,SAA8B;IAE9B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnF,KAAK,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACvD,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;YAAE,SAAS;QAE3C,MAAM,WAAW,GAAc;YAC7B,MAAM,EAAE,mBAAmB,aAAa,EAAE;YAC1C,gBAAgB,EAAE,aAAa;YAC/B,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,EAAE,EAAE,oDAAoD;YAC9D,UAAU,EAAE,EAAE;SACf,CAAC;QAEF,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,+EAA+E;YAC/E,cAAc,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC;gBAClB,aAAa;gBACb,OAAO;gBACP,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YACH,cAAc,CAAC,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/src/snapshot/snapshot-compiler.d.ts

@@ -12,17 +12,8 @@
  */
 import type { Page } from 'puppeteer-core';
 import type { CdpClient } from '../cdp/cdp-client.interface.js';
-import type { BaseSnapshot, SnapshotOptions } from './snapshot.types.js';
-import { type RawNodeData } from './extractors/index.js';
-/**
- * Snapshot compiler options
- */
-export interface CompileOptions extends Partial<SnapshotOptions> {
-    /** Include readable content nodes (headings, paragraphs). Default: true */
-    includeReadable?: boolean;
-    /** Extract bounding boxes and layout info. Default: true */
-    includeLayout?: boolean;
-}
+import type { BaseSnapshot } from './snapshot.types.js';
+import { type CompileOptions } from './kind-mapping.js';
 /**
  * SnapshotCompiler class
  *
@@ -54,32 +45,7 @@ export declare class SnapshotCompiler {
      * Transform raw node data to ReadableNode.
      */
     private transformNode;
-    /**
-     * Get NodeKind from HTML tag name.
-     */
-    private getKindFromTag;
-    /**
-     * Filter out noise nodes to reduce snapshot size.
-     *
-     * Filters:
-     * 1. Empty list/listitem containers with no semantic name AND no interactive descendants
-     * 2. StaticText/text nodes that mirror their parent's label exactly
-     */
-    private filterNoiseNodes;
 }
-/**
- * Slice nodes to max_nodes budget while preserving high z-index overlay content.
- *
- * Portal-rendered content (dropdowns, popovers, modals) appears at the end of
- * DOM order. On heavy pages, a naive slice truncates it.
- *
- * Strategy:
- * 1. Partition nodes into overlay (z-index > threshold) and main
- * 2. Take all overlay nodes (up to 30% of budget)
- * 3. Fill remaining budget with main nodes (DOM order)
- * 4. Re-sort by original DOM order
- */
-export declare function sliceWithOverlayPriority(nodes: RawNodeData[], maxNodes: number): RawNodeData[];
 /**
  * Export a compile function for simpler usage.
  */

package/dist/src/snapshot/snapshot-compiler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-compiler.d.ts","sourceRoot":"","sources":["../../../src/snapshot/snapshot-compiler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,KAAK,EACV,YAAY,EAGZ,eAAe,EAMhB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAYL,KAAK,WAAW,EAOjB,MAAM,uBAAuB,CAAC;AAkS/B;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,OAAO,CAAC,eAAe,CAAC;IAC9D,2EAA2E;IAC3E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4DAA4D;IAC5D,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAwED;;;;GAIG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;gBAEhB,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAI7C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAK1B;;;;;;;OAOG;IACG,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAicjF;;OAEG;YACW,iBAAiB;IAe/B;;OAEG;IACH,OAAO,CAAC,aAAa;IAsHrB;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;CAsGzB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CA2C9F;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,SAAS,EACd,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAChC,OAAO,CAAC,YAAY,CAAC,CAGvB"}
+{"version":3,"file":"snapshot-compiler.d.ts","sourceRoot":"","sources":["../../../src/snapshot/snapshot-compiler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,KAAK,EACV,YAAY,EAQb,MAAM,qBAAqB,CAAC;AAyC7B,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,mBAAmB,CAAC;AAE3B;;;;GAIG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;gBAEhB,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAI7C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAK1B;;;;;;;OAOG;IACG,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA2XjF;;OAEG;YACW,iBAAiB;IAe/B;;OAEG;IACH,OAAO,CAAC,aAAa;CA+HtB;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,SAAS,EACd,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAChC,OAAO,CAAC,YAAY,CAAC,CAGvB"}