@skillful-agents/agent-computer 0.0.4 → 0.0.6

package/dist-cjs/src/cdp/ax-tree.js

@@ -0,0 +1,162 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CDPAXTree = void 0;
+const refs_js_1 = require("../refs.js");
+const role_map_js_1 = require("./role-map.js");
+const MAX_ELEMENTS = 500;
+const MAX_DEPTH = 50;
+class CDPAXTree {
+    connection;
+    constructor(connection) {
+        this.connection = connection;
+    }
+    async getTree(options = {}) {
+        const { depth, interactive = false } = options;
+        const params = {};
+        if (depth !== undefined) {
+            params.depth = depth;
+        }
+        const result = (await this.connection.send('Accessibility.getFullAXTree', params));
+        const nodes = result.nodes;
+        if (!nodes || nodes.length === 0) {
+            return { elements: [], refMap: new Map() };
+        }
+        // Build a lookup map: nodeId → CDPAXNode
+        const nodeMap = new Map();
+        for (const node of nodes) {
+            nodeMap.set(node.nodeId, node);
+        }
+        // Ref counter per prefix
+        const refCounters = {};
+        const refMap = new Map();
+        let totalElements = 0;
+        const buildElement = (node, currentDepth) => {
+            if (totalElements >= MAX_ELEMENTS)
+                return null;
+            if (currentDepth > MAX_DEPTH)
+                return null;
+            // Skip ignored nodes — parent's buildChildren handles grandchild collection
+            if (node.ignored) {
+                return null;
+            }
+            const cdpRole = node.role?.value ?? 'generic';
+            const mappedRole = (0, role_map_js_1.mapCDPRole)(cdpRole);
+            // In interactive mode, check if this node's CDP role qualifies.
+            // We still recurse into children of non-interactive nodes.
+            if (interactive && !role_map_js_1.INTERACTIVE_ROLES.has(cdpRole)) {
+                const children = buildChildren(node, currentDepth);
+                // Promote children of non-interactive nodes
+                if (children.length === 1)
+                    return children[0];
+                if (children.length > 1) {
+                    // Wrap in a group so children aren't lost
+                    const prefix = (0, refs_js_1.roleToPrefix)(mappedRole);
+                    refCounters[prefix] = (refCounters[prefix] ?? 0) + 1;
+                    const ref = `@${prefix}${refCounters[prefix]}`;
+                    totalElements++;
+                    if (node.backendDOMNodeId !== undefined) {
+                        refMap.set(ref, {
+                            nodeId: node.nodeId,
+                            backendDOMNodeId: node.backendDOMNodeId,
+                        });
+                    }
+                    const element = {
+                        ref,
+                        role: mappedRole,
+                        label: node.name?.value ?? null,
+                        value: node.value?.value != null ? String(node.value.value) : null,
+                        enabled: getEnabled(node),
+                        focused: getFocused(node),
+                        bounds: [0, 0, 0, 0],
+                    };
+                    if (children.length > 0) {
+                        element.children = children;
+                    }
+                    return element;
+                }
+                return null;
+            }
+            const prefix = (0, refs_js_1.roleToPrefix)(mappedRole);
+            refCounters[prefix] = (refCounters[prefix] ?? 0) + 1;
+            const ref = `@${prefix}${refCounters[prefix]}`;
+            totalElements++;
+            if (node.backendDOMNodeId !== undefined) {
+                refMap.set(ref, {
+                    nodeId: node.nodeId,
+                    backendDOMNodeId: node.backendDOMNodeId,
+                });
+            }
+            const children = buildChildren(node, currentDepth);
+            const element = {
+                ref,
+                role: mappedRole,
+                label: node.name?.value ?? null,
+                value: node.value?.value != null ? String(node.value.value) : null,
+                enabled: getEnabled(node),
+                focused: getFocused(node),
+                bounds: [0, 0, 0, 0],
+            };
+            if (children.length > 0) {
+                element.children = children;
+            }
+            return element;
+        };
+        const buildChildren = (parent, currentDepth) => {
+            if (!parent.childIds || parent.childIds.length === 0)
+                return [];
+            const children = [];
+            for (const childId of parent.childIds) {
+                if (totalElements >= MAX_ELEMENTS)
+                    break;
+                const childNode = nodeMap.get(childId);
+                if (!childNode)
+                    continue;
+                const el = buildElement(childNode, currentDepth + 1);
+                if (el) {
+                    children.push(el);
+                }
+                else if (childNode.childIds) {
+                    // If the child was ignored/skipped, collect its grandchildren
+                    const grandchildren = buildChildren(childNode, currentDepth + 1);
+                    children.push(...grandchildren);
+                }
+            }
+            return children;
+        };
+        // Root is the first node
+        const root = nodes[0];
+        const rootElement = buildElement(root, 0);
+        const elements = [];
+        if (rootElement) {
+            // If root is a webarea, return its children directly (common pattern)
+            if (rootElement.role === 'webarea' && rootElement.children) {
+                elements.push(...rootElement.children);
+            }
+            else {
+                elements.push(rootElement);
+            }
+        }
+        return { elements, refMap };
+    }
+}
+exports.CDPAXTree = CDPAXTree;
+function getEnabled(node) {
+    if (!node.properties)
+        return true;
+    for (const prop of node.properties) {
+        if (prop.name === 'disabled') {
+            return !prop.value.value;
+        }
+    }
+    return true;
+}
+function getFocused(node) {
+    if (!node.properties)
+        return false;
+    for (const prop of node.properties) {
+        if (prop.name === 'focused') {
+            return !!prop.value.value;
+        }
+    }
+    return false;
+}

package/dist-cjs/src/cdp/bounds.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getBounds = getBounds;
+exports.toScreenCoords = toScreenCoords;
+exports.resolveAllBounds = resolveAllBounds;
+/**
+ * Fetch the CSS bounding box for a DOM node via CDP DOM.getBoxModel.
+ * Returns [x, y, width, height] from the content quad.
+ * Returns [0, 0, 0, 0] if the box model cannot be retrieved.
+ */
+async function getBounds(connection, backendDOMNodeId) {
+    try {
+        const result = (await connection.send('DOM.getBoxModel', {
+            backendNodeId: backendDOMNodeId,
+        }));
+        const content = result.model.content;
+        // Content quad: [x1,y1, x2,y2, x3,y3, x4,y4]
+        const x = content[0];
+        const y = content[1];
+        const w = content[2] - content[0];
+        const h = content[7] - content[1];
+        return [x, y, w, h];
+    }
+    catch {
+        return [0, 0, 0, 0];
+    }
+}
+/**
+ * Convert CSS-pixel bounds to screen coordinates.
+ */
+function toScreenCoords(cssBounds, windowBounds, contentOffset, scaleFactor) {
+    const screenX = (cssBounds[0] + contentOffset.x) * scaleFactor + windowBounds[0];
+    const screenY = (cssBounds[1] + contentOffset.y) * scaleFactor + windowBounds[1];
+    const screenW = cssBounds[2] * scaleFactor;
+    const screenH = cssBounds[3] * scaleFactor;
+    return [screenX, screenY, screenW, screenH];
+}
+const CONCURRENCY_LIMIT = 50;
+/**
+ * Batch-resolve bounds for all elements that have entries in refMap.
+ * Mutates elements in-place, updating their `bounds` field.
+ */
+async function resolveAllBounds(connection, elements, refMap, windowBounds, contentOffset, scaleFactor) {
+    // Collect all (element, nodeRef) pairs by flattening the tree
+    const work = [];
+    const collectWork = (els) => {
+        for (const el of els) {
+            const nodeRef = refMap.get(el.ref);
+            if (nodeRef) {
+                work.push({ element: el, nodeRef });
+            }
+            if (el.children) {
+                collectWork(el.children);
+            }
+        }
+    };
+    collectWork(elements);
+    // Process in batches with concurrency limit
+    for (let i = 0; i < work.length; i += CONCURRENCY_LIMIT) {
+        const batch = work.slice(i, i + CONCURRENCY_LIMIT);
+        await Promise.all(batch.map(async ({ element, nodeRef }) => {
+            const cssBounds = await getBounds(connection, nodeRef.backendDOMNodeId);
+            element.bounds = toScreenCoords(cssBounds, windowBounds, contentOffset, scaleFactor);
+        }));
+    }
+}

package/dist-cjs/src/cdp/client.js

@@ -0,0 +1,272 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CDPClient = void 0;
+const connection_js_1 = require("./connection.js");
+const discovery_js_1 = require("./discovery.js");
+const ax_tree_js_1 = require("./ax-tree.js");
+const interactions_js_1 = require("./interactions.js");
+const bounds_js_1 = require("./bounds.js");
+const diff_js_1 = require("./diff.js");
+class CDPClient {
+    port;
+    connection;
+    axTree = null;
+    interactions = null;
+    lastRefMap = new Map();
+    lastSnapshotId = null;
+    lastElements = [];
+    contentOffset = { x: 0, y: 0 };
+    scaleFactor = 2; // Retina default
+    constructor(port) {
+        this.port = port;
+        this.connection = new connection_js_1.CDPConnection();
+    }
+    /** Connect to CDP target, enable domains */
+    async connect() {
+        const target = await (0, discovery_js_1.waitForCDP)(this.port, 10000);
+        await this.connection.connect(target.webSocketDebuggerUrl);
+        await this.enableDomains();
+    }
+    /** Reconnect after WebSocket drop */
+    async reconnect() {
+        try {
+            await this.connection.close();
+        }
+        catch { /* ok */ }
+        const target = await (0, discovery_js_1.waitForCDP)(this.port, 10000);
+        this.connection = new connection_js_1.CDPConnection();
+        await this.connection.connect(target.webSocketDebuggerUrl);
+        await this.enableDomains();
+    }
+    /** Disconnect from CDP */
+    async disconnect() {
+        await this.connection.close();
+        this.axTree = null;
+        this.interactions = null;
+    }
+    async enableDomains() {
+        await Promise.all([
+            this.connection.send('Accessibility.enable'),
+            this.connection.send('DOM.enable'),
+            this.connection.send('Page.enable'),
+            this.connection.send('Runtime.enable'),
+        ]);
+        this.axTree = new ax_tree_js_1.CDPAXTree(this.connection);
+        this.interactions = new interactions_js_1.CDPInteractions(this.connection);
+        // Determine content offset once per connection
+        await this.updateContentOffset();
+    }
+    /** Check if connected */
+    isConnected() {
+        return this.connection.connected;
+    }
+    /** Take a snapshot of the accessibility tree */
+    async snapshot(options = {}, windowInfo) {
+        if (!this.axTree)
+            throw new Error('CDPClient not connected');
+        const { elements, refMap } = await this.axTree.getTree({
+            interactive: options.interactive,
+            depth: options.depth,
+        });
+        // Resolve bounds
+        await (0, bounds_js_1.resolveAllBounds)(this.connection, elements, refMap, windowInfo.bounds, this.contentOffset, this.scaleFactor);
+        this.lastRefMap = refMap;
+        this.lastSnapshotId = `cdp-${Date.now()}`;
+        this.lastElements = elements;
+        return {
+            snapshot_id: this.lastSnapshotId,
+            window: windowInfo,
+            elements,
+            fallback: null,
+        };
+    }
+    /** Click an element by ref */
+    async click(ref, options = {}) {
+        const nodeRef = this.resolveRef(ref);
+        const bounds = await this.getCSSBounds(nodeRef.backendDOMNodeId);
+        await this.interactions.click(nodeRef.backendDOMNodeId, bounds, options);
+    }
+    /** Click at CSS viewport coordinates */
+    async clickAt(x, y, options = {}) {
+        await this.interactions.clickAt(x, y, options);
+    }
+    /** Hover over an element */
+    async hover(ref) {
+        const nodeRef = this.resolveRef(ref);
+        const bounds = await this.getCSSBounds(nodeRef.backendDOMNodeId);
+        await this.interactions.hover(bounds);
+    }
+    /** Focus an element */
+    async focus(ref) {
+        const nodeRef = this.resolveRef(ref);
+        await this.interactions.focus(nodeRef.backendDOMNodeId);
+    }
+    /** Type text */
+    async type(text, options = {}) {
+        if (options.delay) {
+            await this.interactions.typeWithDelay(text, options.delay);
+        }
+        else {
+            await this.interactions.type(text);
+        }
+    }
+    /** Fill an element with text */
+    async fill(ref, text) {
+        const nodeRef = this.resolveRef(ref);
+        await this.interactions.fill(nodeRef.backendDOMNodeId, text);
+    }
+    /** Press a key combination */
+    async key(combo, repeat) {
+        await this.interactions.key(combo, repeat);
+    }
+    /** Scroll in a direction */
+    async scroll(direction, options = {}) {
+        let atX;
+        let atY;
+        if (options.on) {
+            const nodeRef = this.resolveRef(options.on);
+            const bounds = await this.getCSSBounds(nodeRef.backendDOMNodeId);
+            atX = bounds[0] + bounds[2] / 2;
+            atY = bounds[1] + bounds[3] / 2;
+        }
+        await this.interactions.scroll(direction, options.amount, atX, atY);
+    }
+    /** Select a value in a dropdown */
+    async select(ref, value) {
+        const nodeRef = this.resolveRef(ref);
+        await this.interactions.select(nodeRef.backendDOMNodeId, value);
+    }
+    /** Check a checkbox */
+    async check(ref) {
+        const nodeRef = this.resolveRef(ref);
+        const bounds = await this.getCSSBounds(nodeRef.backendDOMNodeId);
+        await this.interactions.check(nodeRef.backendDOMNodeId, bounds);
+    }
+    /** Uncheck a checkbox */
+    async uncheck(ref) {
+        const nodeRef = this.resolveRef(ref);
+        const bounds = await this.getCSSBounds(nodeRef.backendDOMNodeId);
+        await this.interactions.uncheck(nodeRef.backendDOMNodeId, bounds);
+    }
+    /** Check if UI changed since last snapshot */
+    async changed() {
+        if (!this.axTree || this.lastElements.length === 0)
+            return true;
+        const { elements } = await this.axTree.getTree({});
+        return (0, diff_js_1.computeChanged)(this.lastElements, elements);
+    }
+    /** Get diff since last snapshot */
+    async diff() {
+        if (!this.axTree || this.lastElements.length === 0) {
+            return { changed: true, added: [], removed: [], modified: [] };
+        }
+        const { elements } = await this.axTree.getTree({});
+        return (0, diff_js_1.computeDiff)(this.lastElements, elements);
+    }
+    /** Find elements by text in last snapshot */
+    find(text, options = {}) {
+        const lowerText = text.toLowerCase();
+        let results = this.flattenElements(this.lastElements).filter(el => {
+            const matchText = (el.label?.toLowerCase().includes(lowerText)) ||
+                (el.value?.toLowerCase().includes(lowerText));
+            if (!matchText)
+                return false;
+            if (options.role && el.role !== options.role)
+                return false;
+            return true;
+        });
+        if (options.first && results.length > 0) {
+            results = [results[0]];
+        }
+        return { elements: results };
+    }
+    /** Read element value from last snapshot */
+    read(ref, attr) {
+        const el = this.findElementByRef(ref);
+        if (!el)
+            throw new Error(`Element not found: ${ref}`);
+        if (attr === 'label')
+            return { ref, value: el.label };
+        if (attr === 'role')
+            return { ref, value: el.role };
+        if (attr === 'enabled')
+            return { ref, value: el.enabled };
+        if (attr === 'focused')
+            return { ref, value: el.focused };
+        return { ref, value: el.value };
+    }
+    /** Get element bounds */
+    async box(ref) {
+        const el = this.findElementByRef(ref);
+        if (!el)
+            throw new Error(`Element not found: ${ref}`);
+        return { ref, bounds: el.bounds };
+    }
+    /** Check element state */
+    is(state, ref) {
+        const el = this.findElementByRef(ref);
+        if (!el)
+            throw new Error(`Element not found: ${ref}`);
+        switch (state) {
+            case 'enabled': return { state, value: el.enabled };
+            case 'focused': return { state, value: el.focused };
+            case 'visible': return { state, value: el.bounds[2] > 0 && el.bounds[3] > 0 };
+            default: return { state, value: false };
+        }
+    }
+    /** Get children of an element from last snapshot */
+    children(ref) {
+        const el = this.findElementByRef(ref);
+        if (!el)
+            throw new Error(`Element not found: ${ref}`);
+        return { ref, children: el.children ?? [] };
+    }
+    /** Get the underlying connection (for advanced usage) */
+    getConnection() {
+        return this.connection;
+    }
+    /** Get the last ref map */
+    getLastRefMap() {
+        return this.lastRefMap;
+    }
+    // --- Private helpers ---
+    resolveRef(ref) {
+        const nodeRef = this.lastRefMap.get(ref);
+        if (!nodeRef) {
+            throw new Error(`CDP ref not found: ${ref}. Take a snapshot first.`);
+        }
+        return nodeRef;
+    }
+    async getCSSBounds(backendDOMNodeId) {
+        return (0, bounds_js_1.getBounds)(this.connection, backendDOMNodeId);
+    }
+    async updateContentOffset() {
+        try {
+            const result = await this.connection.send('Runtime.evaluate', {
+                expression: 'JSON.stringify({ x: window.screenX, y: window.screenY })',
+                returnByValue: true,
+            });
+            const parsed = JSON.parse(result.result.value);
+            this.contentOffset = { x: parsed.x ?? 0, y: parsed.y ?? 0 };
+        }
+        catch {
+            this.contentOffset = { x: 0, y: 0 };
+        }
+    }
+    flattenElements(elements) {
+        const result = [];
+        const walk = (els) => {
+            for (const el of els) {
+                result.push(el);
+                if (el.children)
+                    walk(el.children);
+            }
+        };
+        walk(elements);
+        return result;
+    }
+    findElementByRef(ref) {
+        return this.flattenElements(this.lastElements).find(el => el.ref === ref);
+    }
+}
+exports.CDPClient = CDPClient;