@skillful-agents/agent-computer 0.0.4 → 0.0.6

package/dist-cjs/src/cdp/interactions.js

@@ -0,0 +1,301 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CDPInteractions = void 0;
+// Modifier flag values for CDP Input events
+const MODIFIER_FLAGS = {
+    alt: 1,
+    ctrl: 2,
+    meta: 4, // Cmd on Mac
+    shift: 8,
+};
+// Key name mapping for special keys
+const KEY_MAP = {
+    enter: { key: 'Enter', code: 'Enter', keyCode: 13 },
+    return: { key: 'Enter', code: 'Enter', keyCode: 13 },
+    tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
+    escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
+    esc: { key: 'Escape', code: 'Escape', keyCode: 27 },
+    backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
+    delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
+    space: { key: ' ', code: 'Space', keyCode: 32 },
+    up: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
+    down: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
+    left: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
+    right: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
+    home: { key: 'Home', code: 'Home', keyCode: 36 },
+    end: { key: 'End', code: 'End', keyCode: 35 },
+    pageup: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
+    pagedown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
+    // Letters a-z will be handled dynamically
+};
+class CDPInteractions {
+    connection;
+    constructor(connection) {
+        this.connection = connection;
+    }
+    /** Click at element center (CSS viewport coords). Options: right, double, modifiers */
+    async click(backendDOMNodeId, cssBounds, options = {}) {
+        const x = cssBounds[0] + cssBounds[2] / 2;
+        const y = cssBounds[1] + cssBounds[3] / 2;
+        await this.clickAt(x, y, options);
+    }
+    /** Click at specific CSS viewport coords */
+    async clickAt(x, y, options = {}) {
+        const button = options.right ? 'right' : 'left';
+        const clickCount = options.double ? 2 : (options.count ?? 1);
+        const modifiers = this.computeModifiers(options.modifiers);
+        // Move mouse first
+        await this.connection.send('Input.dispatchMouseEvent', {
+            type: 'mouseMoved',
+            x, y,
+            modifiers,
+        });
+        // For double click, send two full click sequences
+        for (let i = 0; i < clickCount; i++) {
+            await this.connection.send('Input.dispatchMouseEvent', {
+                type: 'mousePressed',
+                x, y,
+                button,
+                clickCount: i + 1,
+                modifiers,
+            });
+            await this.connection.send('Input.dispatchMouseEvent', {
+                type: 'mouseReleased',
+                x, y,
+                button,
+                clickCount: i + 1,
+                modifiers,
+            });
+        }
+    }
+    /** Hover over element (move mouse to center) */
+    async hover(cssBounds) {
+        const x = cssBounds[0] + cssBounds[2] / 2;
+        const y = cssBounds[1] + cssBounds[3] / 2;
+        await this.connection.send('Input.dispatchMouseEvent', {
+            type: 'mouseMoved',
+            x, y,
+        });
+    }
+    /** Focus an element by backendNodeId */
+    async focus(backendDOMNodeId) {
+        await this.connection.send('DOM.focus', { backendNodeId: backendDOMNodeId });
+    }
+    /** Insert text at current focus (fast, no key events) */
+    async type(text) {
+        await this.connection.send('Input.insertText', { text });
+    }
+    /** Type with per-character key events and optional delay */
+    async typeWithDelay(text, delay) {
+        for (const char of text) {
+            await this.connection.send('Input.dispatchKeyEvent', {
+                type: 'keyDown',
+                key: char,
+                text: char,
+            });
+            await this.connection.send('Input.dispatchKeyEvent', {
+                type: 'keyUp',
+                key: char,
+            });
+            if (delay > 0) {
+                await sleep(delay);
+            }
+        }
+    }
+    /** Fill: focus element, select all, insert text. Fallback: set .value via Runtime */
+    async fill(backendDOMNodeId, text) {
+        // Focus the element
+        await this.focus(backendDOMNodeId);
+        await sleep(50);
+        // Select all (Cmd+A on Mac)
+        await this.key('cmd+a');
+        await sleep(50);
+        // Insert text (replaces selection)
+        await this.type(text);
+    }
+    /** Press a key combination (e.g., "cmd+a", "shift+enter", "ctrl+c") */
+    async key(combo, repeat) {
+        const times = repeat ?? 1;
+        const { modifiers, keyInfo } = this.parseCombo(combo);
+        for (let i = 0; i < times; i++) {
+            // Press modifier keys
+            for (const mod of modifiers) {
+                await this.connection.send('Input.dispatchKeyEvent', {
+                    type: 'rawKeyDown',
+                    key: this.modifierKeyName(mod),
+                    modifiers: this.computeModifiers(modifiers),
+                });
+            }
+            // Press the main key
+            const modifierFlags = this.computeModifiers(modifiers);
+            await this.connection.send('Input.dispatchKeyEvent', {
+                type: 'rawKeyDown',
+                key: keyInfo.key,
+                code: keyInfo.code,
+                windowsVirtualKeyCode: keyInfo.keyCode,
+                modifiers: modifierFlags,
+            });
+            // If it's a printable character with no modifiers (or just shift), send char event
+            if (keyInfo.key.length === 1 && modifiers.every(m => m === 'shift')) {
+                await this.connection.send('Input.dispatchKeyEvent', {
+                    type: 'char',
+                    key: keyInfo.key,
+                    text: keyInfo.key,
+                    modifiers: modifierFlags,
+                });
+            }
+            await this.connection.send('Input.dispatchKeyEvent', {
+                type: 'keyUp',
+                key: keyInfo.key,
+                code: keyInfo.code,
+                windowsVirtualKeyCode: keyInfo.keyCode,
+                modifiers: modifierFlags,
+            });
+            // Release modifier keys (reverse order)
+            for (const mod of [...modifiers].reverse()) {
+                await this.connection.send('Input.dispatchKeyEvent', {
+                    type: 'keyUp',
+                    key: this.modifierKeyName(mod),
+                    modifiers: 0,
+                });
+            }
+            if (i < times - 1)
+                await sleep(50);
+        }
+    }
+    /** Scroll using mouseWheel events */
+    async scroll(direction, amount = 3, atX, atY) {
+        const x = atX ?? 300; // center of typical window
+        const y = atY ?? 250;
+        const pixelAmount = amount * 100; // Each "amount" unit = 100px
+        let deltaX = 0;
+        let deltaY = 0;
+        switch (direction) {
+            case 'up':
+                deltaY = -pixelAmount;
+                break;
+            case 'down':
+                deltaY = pixelAmount;
+                break;
+            case 'left':
+                deltaX = -pixelAmount;
+                break;
+            case 'right':
+                deltaX = pixelAmount;
+                break;
+        }
+        await this.connection.send('Input.dispatchMouseEvent', {
+            type: 'mouseWheel',
+            x, y,
+            deltaX,
+            deltaY,
+        });
+    }
+    /** Select a value in a <select> element */
+    async select(backendDOMNodeId, value) {
+        // Resolve to a remote object first
+        const { object } = await this.connection.send('DOM.resolveNode', {
+            backendNodeId: backendDOMNodeId,
+        });
+        await this.connection.send('Runtime.callFunctionOn', {
+            objectId: object.objectId,
+            functionDeclaration: `function(val) {
+        for (let i = 0; i < this.options.length; i++) {
+          if (this.options[i].value === val || this.options[i].text === val) {
+            this.selectedIndex = i;
+            this.dispatchEvent(new Event('change', { bubbles: true }));
+            return true;
+          }
+        }
+        return false;
+      }`,
+            arguments: [{ value }],
+        });
+    }
+    /** Check a checkbox (click if not already checked) */
+    async check(backendDOMNodeId, cssBounds) {
+        const checked = await this.getCheckedState(backendDOMNodeId);
+        if (!checked) {
+            await this.click(backendDOMNodeId, cssBounds);
+        }
+    }
+    /** Uncheck a checkbox (click if currently checked) */
+    async uncheck(backendDOMNodeId, cssBounds) {
+        const checked = await this.getCheckedState(backendDOMNodeId);
+        if (checked) {
+            await this.click(backendDOMNodeId, cssBounds);
+        }
+    }
+    /** Get .checked state of an element via Runtime */
+    async getCheckedState(backendDOMNodeId) {
+        const { object } = await this.connection.send('DOM.resolveNode', {
+            backendNodeId: backendDOMNodeId,
+        });
+        const result = await this.connection.send('Runtime.callFunctionOn', {
+            objectId: object.objectId,
+            functionDeclaration: 'function() { return this.checked; }',
+            returnByValue: true,
+        });
+        return result.result.value === true;
+    }
+    /** Parse key combo string like "cmd+shift+a" */
+    parseCombo(combo) {
+        const parts = combo.toLowerCase().split('+');
+        const modifiers = [];
+        let mainKey = '';
+        for (const part of parts) {
+            const p = part.trim();
+            if (p === 'cmd' || p === 'command' || p === 'meta') {
+                modifiers.push('meta');
+            }
+            else if (p === 'ctrl' || p === 'control') {
+                modifiers.push('ctrl');
+            }
+            else if (p === 'alt' || p === 'option' || p === 'opt') {
+                modifiers.push('alt');
+            }
+            else if (p === 'shift') {
+                modifiers.push('shift');
+            }
+            else {
+                mainKey = p;
+            }
+        }
+        const keyInfo = KEY_MAP[mainKey] ?? this.letterKeyInfo(mainKey);
+        return { modifiers, keyInfo };
+    }
+    letterKeyInfo(key) {
+        if (key.length === 1) {
+            const upper = key.toUpperCase();
+            return {
+                key: key,
+                code: `Key${upper}`,
+                keyCode: upper.charCodeAt(0),
+            };
+        }
+        // Unknown key - pass through
+        return { key, code: key, keyCode: 0 };
+    }
+    modifierKeyName(mod) {
+        switch (mod) {
+            case 'meta': return 'Meta';
+            case 'ctrl': return 'Control';
+            case 'alt': return 'Alt';
+            case 'shift': return 'Shift';
+            default: return mod;
+        }
+    }
+    computeModifiers(mods) {
+        if (!mods)
+            return 0;
+        let flags = 0;
+        for (const mod of mods) {
+            flags |= MODIFIER_FLAGS[mod] ?? 0;
+        }
+        return flags;
+    }
+}
+exports.CDPInteractions = CDPInteractions;
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}

package/dist-cjs/src/cdp/port-manager.js

@@ -0,0 +1,68 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findFreePort = findFreePort;
+const net = __importStar(require("net"));
+const PORT_RANGE_START = 19200;
+const PORT_RANGE_END = 19299;
+/** Find a free port in the CDP range by probing for one not in use. */
+async function findFreePort() {
+    for (let port = PORT_RANGE_START; port <= PORT_RANGE_END; port++) {
+        if (await isPortAvailable(port)) {
+            return port;
+        }
+    }
+    throw new Error(`No available CDP ports in range ${PORT_RANGE_START}-${PORT_RANGE_END}`);
+}
+/** Check if a port is available by attempting a TCP connection */
+function isPortAvailable(port) {
+    return new Promise((resolve) => {
+        const socket = new net.Socket();
+        socket.setTimeout(500);
+        socket.on('connect', () => {
+            socket.destroy();
+            resolve(false); // Port is in use
+        });
+        socket.on('timeout', () => {
+            socket.destroy();
+            resolve(true); // Timeout = nothing listening = available
+        });
+        socket.on('error', () => {
+            socket.destroy();
+            resolve(true); // Connection refused = available
+        });
+        socket.connect(port, '127.0.0.1');
+    });
+}

package/dist-cjs/src/cdp/role-map.js

@@ -0,0 +1,102 @@
+"use strict";
+// Maps CDP ARIA roles to normalized roles used by the agent-computer Element format.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INTERACTIVE_ROLES = void 0;
+exports.mapCDPRole = mapCDPRole;
+const CDP_ROLE_MAP = {
+    // Interactive controls
+    button: 'button',
+    textbox: 'textfield',
+    searchbox: 'textfield',
+    textfield: 'textfield',
+    link: 'link',
+    checkbox: 'checkbox',
+    radio: 'radio',
+    slider: 'slider',
+    spinbutton: 'slider',
+    combobox: 'dropdown',
+    listbox: 'dropdown',
+    popupbutton: 'dropdown',
+    // Media
+    img: 'image',
+    image: 'image',
+    // Text content
+    heading: 'text',
+    StaticText: 'text',
+    LabelText: 'text',
+    paragraph: 'text',
+    status: 'text',
+    // Tabs
+    tab: 'tab',
+    tablist: 'tabgroup',
+    tabpanel: 'group',
+    // Menus
+    menuitem: 'menuitem',
+    menu: 'group',
+    menubar: 'group',
+    // Tables
+    table: 'table',
+    grid: 'table',
+    row: 'row',
+    cell: 'row',
+    gridcell: 'row',
+    columnheader: 'row',
+    rowheader: 'row',
+    // Trees
+    tree: 'treeview',
+    treeitem: 'menuitem',
+    // Misc widgets
+    scrollbar: 'scrollarea',
+    progressbar: 'progress',
+    // Generic / presentational
+    generic: 'generic',
+    none: 'generic',
+    presentation: 'generic',
+    separator: 'generic',
+    // Document / web areas
+    WebArea: 'webarea',
+    RootWebArea: 'webarea',
+    document: 'webarea',
+    // Landmark & structural → group
+    navigation: 'group',
+    region: 'group',
+    main: 'group',
+    form: 'group',
+    section: 'group',
+    article: 'group',
+    banner: 'group',
+    complementary: 'group',
+    contentinfo: 'group',
+    list: 'group',
+    listitem: 'group',
+    group: 'group',
+    toolbar: 'group',
+    dialog: 'group',
+    alertdialog: 'group',
+    alert: 'group',
+};
+/** Map a CDP ARIA role string to a normalized role. Unknown roles become 'generic'. */
+function mapCDPRole(cdpRole) {
+    return CDP_ROLE_MAP[cdpRole] ?? 'generic';
+}
+/**
+ * CDP roles considered interactive — used to filter the AX tree in interactive mode.
+ * These are CDP/ARIA role strings (keys in CDP_ROLE_MAP), not normalized roles.
+ */
+exports.INTERACTIVE_ROLES = new Set([
+    'button',
+    'textbox',
+    'searchbox',
+    'textfield',
+    'link',
+    'checkbox',
+    'radio',
+    'slider',
+    'spinbutton',
+    'combobox',
+    'listbox',
+    'popupbutton',
+    'tab',
+    'menuitem',
+    'treeitem',
+]);

package/dist-cjs/src/cdp/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist-cjs/src/cli/commands/apps.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commands_js_1 = require("../commands.js");
+(0, commands_js_1.registerCommand)('apps', async (args, bridge) => {
+    const params = {};
+    if (args.flags['running'])
+        params.running = true;
+    const result = await bridge.send('apps', params);
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('launch', async (args, bridge) => {
+    const name = args.positional[0];
+    if (!name) {
+        return { data: { error: 'Usage: agent-computer launch <name> [--wait] [--background]' }, exitCode: 1 };
+    }
+    const params = {
+        name,
+        wait: args.flags['wait'] === true,
+        background: args.flags['background'] === true,
+    };
+    if (args.flags['open']) {
+        params.open = args.flags['open'];
+    }
+    const result = await bridge.send('launch', params);
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('relaunch', async (args, bridge) => {
+    const name = args.positional[0];
+    if (!name) {
+        return { data: { error: 'Usage: agent-computer relaunch <name>' }, exitCode: 1 };
+    }
+    const result = await bridge.send('relaunch', { name });
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('quit', async (args, bridge) => {
+    const name = args.positional[0];
+    if (!name) {
+        return { data: { error: 'Usage: agent-computer quit <name> [--force]' }, exitCode: 1 };
+    }
+    const result = await bridge.send('quit', { name, force: args.flags['force'] === true });
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('hide', async (args, bridge) => {
+    const name = args.positional[0];
+    if (!name)
+        return { data: { error: 'Usage: agent-computer hide <name>' }, exitCode: 1 };
+    const result = await bridge.send('hide', { name });
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('unhide', async (args, bridge) => {
+    const name = args.positional[0];
+    if (!name)
+        return { data: { error: 'Usage: agent-computer unhide <name>' }, exitCode: 1 };
+    const result = await bridge.send('unhide', { name });
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('switch', async (args, bridge) => {
+    const name = args.positional[0];
+    if (!name)
+        return { data: { error: 'Usage: agent-computer switch <name>' }, exitCode: 1 };
+    const result = await bridge.send('switch', { name });
+    return { data: result, exitCode: 0 };
+});

package/dist-cjs/src/cli/commands/batch.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commands_js_1 = require("../commands.js");
+(0, commands_js_1.registerCommand)('batch', async (args, bridge) => {
+    const jsonStr = args.positional[0] || args.subcommand;
+    if (!jsonStr) {
+        return { data: { error: 'Usage: agent-computer batch \'[["method", ...args], ...]\'' }, exitCode: 1 };
+    }
+    let commands;
+    try {
+        commands = JSON.parse(jsonStr);
+        if (!Array.isArray(commands))
+            throw new Error('not array');
+    }
+    catch {
+        return { data: { error: 'Invalid JSON. Expected array of command arrays.' }, exitCode: 1 };
+    }
+    const params = { commands };
+    if (args.flags['no-stop-on-error'])
+        params.stop_on_error = false;
+    const result = await bridge.send('batch', params);
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('changed', async (args, bridge) => {
+    const params = {};
+    if (args.flags['app'])
+        params.app = args.flags['app'];
+    const result = await bridge.send('changed', params);
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('diff', async (args, bridge) => {
+    const params = {};
+    if (args.flags['app'])
+        params.app = args.flags['app'];
+    const result = await bridge.send('diff', params);
+    return { data: result, exitCode: 0 };
+});

package/dist-cjs/src/cli/commands/click.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commands_js_1 = require("../commands.js");
+const parser_js_1 = require("../parser.js");
+(0, commands_js_1.registerCommand)('click', async (args, bridge) => {
+    const sel = args.positional[0];
+    if (!sel) {
+        return { data: { error: 'Usage: agent-computer click <ref|x,y|label> [--right] [--double] [--count N] [--modifiers keys]' }, exitCode: 1 };
+    }
+    const params = {};
+    const parsed = (0, parser_js_1.parseSelector)(sel);
+    if (parsed.type === 'ref') {
+        params.ref = parsed.ref;
+    }
+    else if (parsed.type === 'coords') {
+        params.x = parsed.x;
+        params.y = parsed.y;
+    }
+    else {
+        // Label-based click — not yet implemented, pass as ref for now
+        params.ref = parsed.label;
+    }
+    if (args.flags['right'])
+        params.right = true;
+    if (args.flags['double'])
+        params.double = true;
+    if (args.flags['count'])
+        params.count = parseInt(args.flags['count'], 10);
+    if (args.flags['modifiers']) {
+        const mods = args.flags['modifiers'];
+        params.modifiers = mods.split(',').map(m => m.trim());
+    }
+    if (args.flags['wait'])
+        params.wait = true;
+    if (args.flags['human'])
+        params.human = true;
+    const result = await bridge.send('click', params);
+    return { data: result, exitCode: 0 };
+});
+(0, commands_js_1.registerCommand)('hover', async (args, bridge) => {
+    const sel = args.positional[0];
+    if (!sel) {
+        return { data: { error: 'Usage: agent-computer hover <ref|x,y>' }, exitCode: 1 };
+    }
+    const params = {};
+    const parsed = (0, parser_js_1.parseSelector)(sel);
+    if (parsed.type === 'ref') {
+        params.ref = parsed.ref;
+    }
+    else if (parsed.type === 'coords') {
+        params.x = parsed.x;
+        params.y = parsed.y;
+    }
+    else {
+        params.ref = parsed.label;
+    }
+    if (args.flags['human'])
+        params.human = true;
+    const result = await bridge.send('hover', params);
+    return { data: result, exitCode: 0 };
+});

package/dist-cjs/src/cli/commands/clipboard.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commands_js_1 = require("../commands.js");
+(0, commands_js_1.registerCommand)('clipboard', async (args, bridge) => {
+    const sub = args.subcommand;
+    switch (sub) {
+        case 'set': {
+            const text = args.positional[0];
+            if (!text) {
+                return { data: { error: 'Usage: agent-computer clipboard set <text>' }, exitCode: 1 };
+            }
+            const result = await bridge.send('clipboard_set', { text });
+            return { data: result, exitCode: 0 };
+        }
+        case 'copy': {
+            const result = await bridge.send('clipboard_copy');
+            return { data: result, exitCode: 0 };
+        }
+        case 'paste': {
+            const result = await bridge.send('paste', { text: '' });
+            // Actually clipboard paste is just Cmd+V
+            const pasteResult = await bridge.send('key', { combo: 'cmd+v' });
+            return { data: pasteResult, exitCode: 0 };
+        }
+        default: {
+            // Read clipboard
+            const result = await bridge.send('clipboard_read');
+            return { data: result, exitCode: 0 };
+        }
+    }
+});