@amaster.ai/pi-computer-use 0.1.1-beta.0

package/dist/actions.js

@@ -0,0 +1,45 @@
+export async function dispatchAction(client, action) {
+    switch (action.type) {
+        case 'screenshot':
+            return undefined;
+        case 'click': {
+            const cmd = action.button === 'right' ? 'right_click' : 'left_click';
+            await client.sendCommand(cmd, { x: action.x, y: action.y });
+            return `Clicked (${action.button ?? 'left'}) at (${action.x}, ${action.y})`;
+        }
+        case 'double_click':
+            await client.sendCommand('double_click', { x: action.x, y: action.y });
+            return `Double-clicked at (${action.x}, ${action.y})`;
+        case 'type':
+            await client.sendCommand('type_text', { text: action.text });
+            return `Typed "${action.text}"`;
+        case 'keypress':
+            await client.sendCommand('hotkey', { keys: action.keys });
+            return `Pressed keys: ${action.keys?.join('+')}`;
+        case 'scroll':
+            await client.sendCommand('scroll', {
+                x: action.scroll_x ?? 0,
+                y: action.scroll_y ?? 0,
+            });
+            return `Scrolled (${action.scroll_x ?? 0}, ${action.scroll_y ?? 0})`;
+        case 'move':
+            await client.sendCommand('move_cursor', { x: action.x, y: action.y });
+            return `Moved cursor to (${action.x}, ${action.y})`;
+        case 'drag':
+            await client.sendCommand('drag', { path: action.path, button: action.button ?? 'left' });
+            return `Dragged along ${action.path?.length ?? 0} points`;
+        case 'wait':
+            await new Promise((r) => setTimeout(r, 1_000));
+            return 'Waited 1 second';
+        case 'run_command': {
+            const result = await client.sendCommand('run_command', { command: action.command });
+            const stdout = result.stdout ?? '';
+            const stderr = result.stderr ?? '';
+            const output = [stdout, stderr].filter(Boolean).join('\n');
+            return output || 'Command executed (no output)';
+        }
+        default:
+            throw new Error(`Unknown action type: ${action.type}`);
+    }
+}
+//# sourceMappingURL=actions.js.map

package/dist/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actions.js","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAeA,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,MAAsB;IAEtB,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,YAAY;YACf,OAAO,SAAS,CAAC;QAEnB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC;YACrE,MAAM,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;YAC5D,OAAO,YAAY,MAAM,CAAC,MAAM,IAAI,MAAM,SAAS,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC;QAC9E,CAAC;QAED,KAAK,cAAc;YACjB,MAAM,MAAM,CAAC,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;YACvE,OAAO,sBAAsB,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC;QAExD,KAAK,MAAM;YACT,MAAM,MAAM,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,OAAO,UAAU,MAAM,CAAC,IAAI,GAAG,CAAC;QAElC,KAAK,UAAU;YACb,MAAM,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,OAAO,iBAAiB,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAEnD,KAAK,QAAQ;YACX,MAAM,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE;gBACjC,CAAC,EAAE,MAAM,CAAC,QAAQ,IAAI,CAAC;gBACvB,CAAC,EAAE,MAAM,CAAC,QAAQ,IAAI,CAAC;aACxB,CAAC,CAAC;YACH,OAAO,aAAa,MAAM,CAAC,QAAQ,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC;QAEvE,KAAK,MAAM;YACT,MAAM,MAAM,CAAC,WAAW,CAAC,aAAa,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;YACtE,OAAO,oBAAoB,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC;QAEtD,KAAK,MAAM;YACT,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC,CAAC;YACzF,OAAO,iBAAiB,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC;QAE5D,KAAK,MAAM;YACT,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,iBAAiB,CAAC;QAE3B,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACpF,MAAM,MAAM,GAAI,MAAM,CAAC,MAAiB,IAAI,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAI,MAAM,CAAC,MAAiB,IAAI,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,OAAO,MAAM,IAAI,8BAA8B,CAAC;QAClD,CAAC;QAED;YACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC"}

package/dist/computer-client.d.ts

@@ -0,0 +1,13 @@
+import type { ComputerUseConfig } from './config.js';
+export declare class ComputerClient {
+    private ws;
+    private commandLock;
+    private config;
+    constructor(config: ComputerUseConfig);
+    connect(): Promise<void>;
+    sendCommand(command: string, params?: Record<string, unknown>): Promise<Record<string, unknown>>;
+    screenshot(): Promise<string>;
+    close(): Promise<void>;
+    private authenticate;
+}
+//# sourceMappingURL=computer-client.d.ts.map

package/dist/computer-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"computer-client.d.ts","sourceRoot":"","sources":["../src/computer-client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAIrD,qBAAa,cAAc;IACzB,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,MAAM,CAAoB;gBAEtB,MAAM,EAAE,iBAAiB;IAI/B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BxB,WAAW,CACf,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GACnC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAqC7B,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAS7B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B,OAAO,CAAC,YAAY;CAyBrB"}

package/dist/computer-client.js

@@ -0,0 +1,109 @@
+import WebSocket from 'ws';
+const COMMAND_TIMEOUT_MS = 30_000;
+export class ComputerClient {
+    ws = null;
+    commandLock = Promise.resolve();
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async connect() {
+        const host = this.config.host ?? '127.0.0.1';
+        const port = this.config.port ?? 8000;
+        const protocol = this.config.apiKey ? 'wss' : 'ws';
+        const url = `${protocol}://${host}:${port}/ws`;
+        const headers = {};
+        if (this.config.apiKey)
+            headers['X-API-Key'] = this.config.apiKey;
+        if (this.config.vmName)
+            headers['X-VM-Name'] = this.config.vmName;
+        this.ws = await new Promise((resolve, reject) => {
+            const ws = new WebSocket(url, { headers });
+            ws.on('open', async () => {
+                if (this.config.apiKey && this.config.vmName) {
+                    try {
+                        await this.authenticate(ws);
+                    }
+                    catch (err) {
+                        ws.close();
+                        reject(err);
+                        return;
+                    }
+                }
+                resolve(ws);
+            });
+            ws.on('error', (err) => reject(new Error(`WebSocket connection failed: ${err.message}`)));
+        });
+    }
+    async sendCommand(command, params = {}) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error('WebSocket not connected');
+        }
+        const result = await new Promise((resolve, reject) => {
+            const execute = () => new Promise((innerResolve, innerReject) => {
+                const timer = setTimeout(() => {
+                    innerReject(new Error(`Command "${command}" timed out after ${COMMAND_TIMEOUT_MS}ms`));
+                }, COMMAND_TIMEOUT_MS);
+                const handler = (data) => {
+                    clearTimeout(timer);
+                    this.ws?.off('message', handler);
+                    try {
+                        const response = JSON.parse(data.toString());
+                        if (response.error) {
+                            innerReject(new Error(response.error));
+                        }
+                        else {
+                            innerResolve(response);
+                        }
+                    }
+                    catch (err) {
+                        innerReject(err);
+                    }
+                };
+                this.ws.on('message', handler);
+                this.ws.send(JSON.stringify({ command, params }));
+            });
+            this.commandLock = this.commandLock.then(() => execute().then(resolve, reject));
+        });
+        return result;
+    }
+    async screenshot() {
+        const response = await this.sendCommand('screenshot', {});
+        const imageData = response.image_data;
+        if (!imageData) {
+            throw new Error('No image_data in screenshot response');
+        }
+        return imageData;
+    }
+    async close() {
+        if (this.ws) {
+            this.ws.close();
+            this.ws = null;
+        }
+    }
+    authenticate(ws) {
+        return new Promise((resolve, reject) => {
+            const handler = (data) => {
+                ws.off('message', handler);
+                try {
+                    const response = JSON.parse(data.toString());
+                    if (response.success) {
+                        resolve();
+                    }
+                    else {
+                        reject(new Error(`Authentication failed: ${response.error ?? 'unknown error'}`));
+                    }
+                }
+                catch (err) {
+                    reject(err);
+                }
+            };
+            ws.on('message', handler);
+            ws.send(JSON.stringify({
+                command: 'authenticate',
+                params: { api_key: this.config.apiKey, container_name: this.config.vmName },
+            }));
+        });
+    }
+}
+//# sourceMappingURL=computer-client.js.map

package/dist/computer-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"computer-client.js","sourceRoot":"","sources":["../src/computer-client.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,IAAI,CAAC;AAG3B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,OAAO,cAAc;IACjB,EAAE,GAAqB,IAAI,CAAC;IAC5B,WAAW,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAClD,MAAM,CAAoB;IAElC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,WAAW,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QACnD,MAAM,GAAG,GAAG,GAAG,QAAQ,MAAM,IAAI,IAAI,IAAI,KAAK,CAAC;QAE/C,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAClE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAElE,IAAI,CAAC,EAAE,GAAG,MAAM,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACzD,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE;gBACvB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC7C,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;oBAC9B,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,CAAC,GAAG,CAAC,CAAC;wBACZ,OAAO;oBACT,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,EAAE,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YACH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5F,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CACf,OAAe,EACf,SAAkC,EAAE;QAEpC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,OAAO,CAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5E,MAAM,OAAO,GAAG,GAAqC,EAAE,CACrD,IAAI,OAAO,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,EAAE;gBACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,WAAW,CAAC,IAAI,KAAK,CAAC,YAAY,OAAO,qBAAqB,kBAAkB,IAAI,CAAC,CAAC,CAAC;gBACzF,CAAC,EAAE,kBAAkB,CAAC,CAAC;gBAEvB,MAAM,OAAO,GAAG,CAAC,IAAuB,EAAE,EAAE;oBAC1C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;oBACjC,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAA4B,CAAC;wBACxE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;4BACnB,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC,CAAC;wBACnD,CAAC;6BAAM,CAAC;4BACN,YAAY,CAAC,QAAQ,CAAC,CAAC;wBACzB,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,WAAW,CAAC,GAAG,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC,CAAC;gBAEF,IAAI,CAAC,EAAG,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;YAEL,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAgC,CAAC;QAC5D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,EAAa;QAChC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,OAAO,GAAG,CAAC,IAAuB,EAAE,EAAE;gBAC1C,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAA4B,CAAC;oBACxE,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;wBACrB,OAAO,EAAE,CAAC;oBACZ,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC,CAAC;oBACnF,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;YACH,CAAC,CAAC;YAEF,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC1B,EAAE,CAAC,IAAI,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,OAAO,EAAE,cAAc;gBACvB,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;aAC5E,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/config.d.ts

@@ -0,0 +1,31 @@
+export interface VisionModelConfig {
+    provider: string;
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+export interface ComputerUseConfig {
+    /** 'managed' spawns cua-computer-server, 'external' connects to existing */
+    mode?: 'managed' | 'external';
+    /** Command to start cua-computer-server (default: 'uvx') */
+    command?: string;
+    /** Package/module to run (default: 'cua-computer-server') */
+    package?: string;
+    /** Extra CLI args for cua-computer-server */
+    extraArgs?: string[];
+    /** Server host (default: '127.0.0.1') */
+    host?: string;
+    /** Server port (default: 8000) */
+    port?: number;
+    /** API key — enables wss + auth for cloud/remote */
+    apiKey?: string;
+    /** VM name for cloud routing */
+    vmName?: string;
+    /** Auto-screenshot after each action (default: true) */
+    autoScreenshot?: boolean;
+    /** Vision model for screenshot analysis */
+    visionModel?: VisionModelConfig;
+}
+export declare function resolveConfig(config?: ComputerUseConfig): ComputerUseConfig;
+export declare function loadConfigFromFile(): ComputerUseConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,4EAA4E;IAC5E,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,CAAC;IAE9B,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,wDAAwD;IACxD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AAWD,wBAAgB,aAAa,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,iBAAiB,CAE3E;AAED,wBAAgB,kBAAkB,IAAI,iBAAiB,CAStD"}

package/dist/config.js

@@ -0,0 +1,25 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+const DEFAULTS = {
+    mode: 'managed',
+    command: 'uvx',
+    package: 'cua-computer-server',
+    host: '127.0.0.1',
+    port: 8000,
+    autoScreenshot: true,
+};
+export function resolveConfig(config) {
+    return { ...DEFAULTS, ...config };
+}
+export function loadConfigFromFile() {
+    try {
+        const configPath = resolve(process.cwd(), 'config.json');
+        const raw = readFileSync(configPath, 'utf-8');
+        const parsed = JSON.parse(raw);
+        return (parsed['pi-computer-use'] ?? {});
+    }
+    catch {
+        return {};
+    }
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmCpC,MAAM,QAAQ,GAA+B;IAC3C,IAAI,EAAE,SAAS;IACf,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,qBAAqB;IAC9B,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,IAAI;IACV,cAAc,EAAE,IAAI;CACrB,CAAC;AAEF,MAAM,UAAU,aAAa,CAAC,MAA0B;IACtD,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;QAC1D,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAsB,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
+import { type ComputerUseConfig, loadConfigFromFile, resolveConfig } from './config.js';
+export type { ComputerUseConfig };
+export { loadConfigFromFile, resolveConfig };
+export default function computerUseExtension(pi: ExtensionAPI): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAoB,MAAM,iCAAiC,CAAC;AAItF,OAAO,EAAE,KAAK,iBAAiB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAIxF,YAAY,EAAE,iBAAiB,EAAE,CAAC;AAClC,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7C,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,IAAI,CA2HnE"}

package/dist/index.js

@@ -0,0 +1,102 @@
+import { Type } from 'typebox';
+import { dispatchAction } from './actions.js';
+import { ComputerClient } from './computer-client.js';
+import { loadConfigFromFile, resolveConfig } from './config.js';
+import { ComputerServerProcess } from './server-process.js';
+import { createPiVisionCaller } from './vision.js';
+export { loadConfigFromFile, resolveConfig };
+export default function computerUseExtension(pi) {
+    const config = resolveConfig(loadConfigFromFile());
+    const serverProcess = new ComputerServerProcess();
+    const client = new ComputerClient(config);
+    let started = false;
+    async function ensureRunning() {
+        if (started)
+            return;
+        if (config.mode !== 'external') {
+            await serverProcess.start(config);
+        }
+        await client.connect();
+        started = true;
+    }
+    pi.registerTool({
+        name: 'computer_use',
+        label: 'computer_use',
+        description: 'Control a computer desktop. Actions: screenshot, click, double_click, type, keypress, scroll, move, drag, wait, run_command. Each call executes one action and returns the resulting screen state.',
+        promptSnippet: 'computer_use — control a desktop: screenshot, click, type, scroll, keypress, drag, run_command',
+        parameters: Type.Object({
+            action: Type.Object({
+                type: Type.String({
+                    description: 'Action type: screenshot | click | double_click | type | keypress | scroll | move | drag | wait | run_command',
+                }),
+                x: Type.Optional(Type.Number({ description: 'X coordinate (click, double_click, move)' })),
+                y: Type.Optional(Type.Number({ description: 'Y coordinate (click, double_click, move)' })),
+                button: Type.Optional(Type.String({ description: 'Mouse button: left | right (default: left)' })),
+                text: Type.Optional(Type.String({ description: 'Text to type (type action)' })),
+                keys: Type.Optional(Type.Array(Type.String(), { description: 'Keys to press (keypress action)' })),
+                scroll_x: Type.Optional(Type.Number({ description: 'Horizontal scroll amount' })),
+                scroll_y: Type.Optional(Type.Number({ description: 'Vertical scroll amount' })),
+                path: Type.Optional(Type.Array(Type.Tuple([Type.Number(), Type.Number()]), {
+                    description: 'Drag path as [[x,y], ...] coordinates',
+                })),
+                command: Type.Optional(Type.String({ description: 'Shell command (run_command action)' })),
+            }),
+        }),
+        async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+            await ensureRunning();
+            if (params.action.type === 'screenshot') {
+                const screenshotBase64 = await client.screenshot();
+                if (config.visionModel) {
+                    const callVision = createPiVisionCaller(config.visionModel, ctx);
+                    const analysis = await callVision('Describe the full screen: identify all visible windows, UI elements, buttons, text fields, and their positions.', screenshotBase64, 'image/png');
+                    return { content: [{ type: 'text', text: analysis }], details: undefined };
+                }
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: 'Screenshot captured (no vision model configured to analyze it).',
+                        },
+                    ],
+                    details: undefined,
+                };
+            }
+            const actionResult = await dispatchAction(client, params.action);
+            if (config.autoScreenshot !== false) {
+                const screenshotBase64 = await client.screenshot();
+                if (config.visionModel) {
+                    const callVision = createPiVisionCaller(config.visionModel, ctx);
+                    const analysis = await callVision('Describe the current screen state after the action. Focus on what changed and what is now visible.', screenshotBase64, 'image/png');
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `${actionResult}\n\nScreen state:\n${analysis}`,
+                            },
+                        ],
+                        details: undefined,
+                    };
+                }
+                return {
+                    content: [{ type: 'text', text: actionResult ?? 'Action executed.' }],
+                    details: undefined,
+                };
+            }
+            return {
+                content: [{ type: 'text', text: actionResult ?? 'Action executed.' }],
+                details: undefined,
+            };
+        },
+    });
+    pi.on('session_start', async () => {
+        // Lazy — actual startup happens on first tool call
+    });
+    pi.on('session_shutdown', async () => {
+        if (started) {
+            await client.close();
+            await serverProcess.stop();
+            started = false;
+        }
+    });
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAuB,cAAc,EAAE,MAAM,cAAc,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAA0B,kBAAkB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAGnD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7C,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAAC,EAAgB;IAC3D,MAAM,MAAM,GAAG,aAAa,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,qBAAqB,EAAE,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,UAAU,aAAa;QAC1B,IAAI,OAAO;YAAE,OAAO;QACpB,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,MAAM,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IAED,EAAE,CAAC,YAAY,CAAC;QACd,IAAI,EAAE,cAAc;QACpB,KAAK,EAAE,cAAc;QACrB,WAAW,EACT,oMAAoM;QACtM,aAAa,EACX,gGAAgG;QAClG,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;gBAClB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;oBAChB,WAAW,EACT,8GAA8G;iBACjH,CAAC;gBACF,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,0CAA0C,EAAE,CAAC,CAAC;gBAC1F,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,0CAA0C,EAAE,CAAC,CAAC;gBAC1F,MAAM,EAAE,IAAI,CAAC,QAAQ,CACnB,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,4CAA4C,EAAE,CAAC,CAC3E;gBACD,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,4BAA4B,EAAE,CAAC,CAAC;gBAC/E,IAAI,EAAE,IAAI,CAAC,QAAQ,CACjB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE,CAAC,CAC9E;gBACD,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,0BAA0B,EAAE,CAAC,CAAC;gBACjF,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,CAAC,CAAC;gBAC/E,IAAI,EAAE,IAAI,CAAC,QAAQ,CACjB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE;oBACrD,WAAW,EAAE,uCAAuC;iBACrD,CAAC,CACH;gBACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,oCAAoC,EAAE,CAAC,CAAC;aAC3F,CAAC;SACH,CAAC;QACF,KAAK,CAAC,OAAO,CACX,WAAmB,EACnB,MAAkC,EAClC,OAAgC,EAChC,SAAkB,EAClB,GAAqB;YAErB,MAAM,aAAa,EAAE,CAAC;YAEtB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACxC,MAAM,gBAAgB,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;gBACnD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;oBACvB,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;oBACjE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAC/B,iHAAiH,EACjH,gBAAgB,EAChB,WAAW,CACZ,CAAC;oBACF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;gBACtF,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,iEAAiE;yBACxE;qBACF;oBACD,OAAO,EAAE,SAAS;iBACnB,CAAC;YACJ,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAEjE,IAAI,MAAM,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;gBACpC,MAAM,gBAAgB,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;gBACnD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;oBACvB,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;oBACjE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAC/B,oGAAoG,EACpG,gBAAgB,EAChB,WAAW,CACZ,CAAC;oBACF,OAAO;wBACL,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAe;gCACrB,IAAI,EAAE,GAAG,YAAY,sBAAsB,QAAQ,EAAE;6BACtD;yBACF;wBACD,OAAO,EAAE,SAAS;qBACnB,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,YAAY,IAAI,kBAAkB,EAAE,CAAC;oBAC9E,OAAO,EAAE,SAAS;iBACnB,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,YAAY,IAAI,kBAAkB,EAAE,CAAC;gBAC9E,OAAO,EAAE,SAAS;aACnB,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,IAAI,EAAE;QAChC,mDAAmD;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QACnC,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;YAC3B,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/server-process.d.ts

@@ -0,0 +1,9 @@
+import type { ComputerUseConfig } from './config.js';
+export declare class ComputerServerProcess {
+    private proc;
+    start(config: ComputerUseConfig): Promise<void>;
+    stop(): Promise<void>;
+    private waitForReady;
+    private probeWs;
+}
+//# sourceMappingURL=server-process.d.ts.map

package/dist/server-process.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server-process.d.ts","sourceRoot":"","sources":["../src/server-process.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAMrD,qBAAa,qBAAqB;IAChC,OAAO,CAAC,IAAI,CAA6B;IAEnC,KAAK,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB/C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAqBb,YAAY;IAoB1B,OAAO,CAAC,OAAO;CAqBhB"}

package/dist/server-process.js

@@ -0,0 +1,76 @@
+import { spawn } from 'node:child_process';
+import WebSocket from 'ws';
+const READY_TIMEOUT_MS = 30_000;
+const POLL_INTERVAL_MS = 500;
+const KILL_GRACE_MS = 2_000;
+export class ComputerServerProcess {
+    proc = null;
+    async start(config) {
+        const cmd = config.command ?? 'uvx';
+        const pkg = config.package ?? 'cua-computer-server';
+        const host = config.host ?? '127.0.0.1';
+        const port = config.port ?? 8000;
+        const args = [pkg, '--host', host, '--port', String(port), ...(config.extraArgs ?? [])];
+        this.proc = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+        this.proc.stderr?.on('data', (chunk) => {
+            const line = chunk.toString().trim();
+            if (line)
+                console.error(`[pi-computer-use] ${line}`);
+        });
+        this.proc.on('error', (err) => {
+            console.error(`[pi-computer-use] Failed to start ${cmd}: ${err.message}`);
+        });
+        await this.waitForReady(host, port);
+    }
+    async stop() {
+        if (!this.proc)
+            return;
+        const proc = this.proc;
+        this.proc = null;
+        proc.kill('SIGTERM');
+        await new Promise((resolve) => {
+            const timer = setTimeout(() => {
+                proc.kill('SIGKILL');
+                resolve();
+            }, KILL_GRACE_MS);
+            proc.on('exit', () => {
+                clearTimeout(timer);
+                resolve();
+            });
+        });
+    }
+    async waitForReady(host, port) {
+        const wsUrl = `ws://${host}:${port}/ws`;
+        const deadline = Date.now() + READY_TIMEOUT_MS;
+        while (Date.now() < deadline) {
+            if (this.proc?.exitCode !== null && this.proc?.exitCode !== undefined) {
+                throw new Error(`cua-computer-server exited with code ${this.proc.exitCode} before becoming ready`);
+            }
+            const ready = await this.probeWs(wsUrl);
+            if (ready)
+                return;
+            await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+        }
+        throw new Error(`cua-computer-server not ready after ${READY_TIMEOUT_MS}ms`);
+    }
+    probeWs(url) {
+        return new Promise((resolve) => {
+            const ws = new WebSocket(url);
+            const timer = setTimeout(() => {
+                ws.terminate();
+                resolve(false);
+            }, 1_000);
+            ws.on('open', () => {
+                clearTimeout(timer);
+                ws.close();
+                resolve(true);
+            });
+            ws.on('error', () => {
+                clearTimeout(timer);
+                ws.terminate();
+                resolve(false);
+            });
+        });
+    }
+}
+//# sourceMappingURL=server-process.js.map

package/dist/server-process.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server-process.js","sourceRoot":"","sources":["../src/server-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,SAAS,MAAM,IAAI,CAAC;AAG3B,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B,MAAM,OAAO,qBAAqB;IACxB,IAAI,GAAwB,IAAI,CAAC;IAEzC,KAAK,CAAC,KAAK,CAAC,MAAyB;QACnC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,IAAI,qBAAqB,CAAC;QACpD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,WAAW,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC;QACjC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;QAExF,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAEpE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,IAAI;gBAAE,OAAO,CAAC,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC5B,OAAO,CAAC,KAAK,CAAC,qCAAqC,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAErB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACrB,OAAO,EAAE,CAAC;YACZ,CAAC,EAAE,aAAa,CAAC,CAAC;YAElB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAY,EAAE,IAAY;QACnD,MAAM,KAAK,GAAG,QAAQ,IAAI,IAAI,IAAI,KAAK,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACtE,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,CAAC,IAAI,CAAC,QAAQ,wBAAwB,CACnF,CAAC;YACJ,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,KAAK;gBAAE,OAAO;YAElB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,uCAAuC,gBAAgB,IAAI,CAAC,CAAC;IAC/E,CAAC;IAEO,OAAO,CAAC,GAAW;QACzB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,EAAE,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,EAAE,KAAK,CAAC,CAAC;YAEV,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAClB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,EAAE,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/vision.d.ts

@@ -0,0 +1,6 @@
+import type { ExtensionContext } from '@earendil-works/pi-coding-agent';
+import type { VisionModelConfig } from './config.js';
+export declare const VISUAL_SYSTEM_PROMPT = "You are a Visual Analysis Agent for computer-use automation.\nYou receive screenshots of a desktop and instructions about what to identify or analyze.\n\nYour responses must:\n- Describe UI elements with their exact pixel coordinates (x, y) relative to the top-left corner (0, 0)\n- Identify clickable elements (buttons, links, inputs, menus) and their bounding regions\n- Describe the current state of the screen (active window, focused element, visible text)\n- Note any dialogs, popups, or overlays that may block interaction\n- Provide spatial relationships between elements\n\nWhen reporting coordinates for click targets, give the CENTER point of the element.\nFormat coordinates as (x, y) inline with descriptions.\n\nIf an element is not visible on screen, explicitly state so.\nDo NOT perform actions \u2014 only analyze and report what you see.";
+export type VisionCaller = (instruction: string, imageBase64: string, mimeType: string) => Promise<string>;
+export declare function createPiVisionCaller(visionConfig: VisionModelConfig, ctx: ExtensionContext): VisionCaller;
+//# sourceMappingURL=vision.d.ts.map

package/dist/vision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../src/vision.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,eAAO,MAAM,oBAAoB,21BAc8B,CAAC;AAEhE,MAAM,MAAM,YAAY,GAAG,CACzB,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC,MAAM,CAAC,CAAC;AAErB,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,iBAAiB,EAC/B,GAAG,EAAE,gBAAgB,GACpB,YAAY,CA+Cd"}

package/dist/vision.js

@@ -0,0 +1,57 @@
+import { complete } from '@earendil-works/pi-ai';
+export const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent for computer-use automation.
+You receive screenshots of a desktop and instructions about what to identify or analyze.
+
+Your responses must:
+- Describe UI elements with their exact pixel coordinates (x, y) relative to the top-left corner (0, 0)
+- Identify clickable elements (buttons, links, inputs, menus) and their bounding regions
+- Describe the current state of the screen (active window, focused element, visible text)
+- Note any dialogs, popups, or overlays that may block interaction
+- Provide spatial relationships between elements
+
+When reporting coordinates for click targets, give the CENTER point of the element.
+Format coordinates as (x, y) inline with descriptions.
+
+If an element is not visible on screen, explicitly state so.
+Do NOT perform actions — only analyze and report what you see.`;
+export function createPiVisionCaller(visionConfig, ctx) {
+    return async (instruction, imageBase64, mimeType) => {
+        const model = ctx.modelRegistry.find(visionConfig.provider, visionConfig.model);
+        if (!model) {
+            throw new Error(`Vision model "${visionConfig.provider}/${visionConfig.model}" not found in model registry.`);
+        }
+        const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+        if (!auth.ok) {
+            throw new Error(`Auth failed for vision model: ${auth.error}`);
+        }
+        const options = {
+            temperature: 0,
+            maxTokens: 2048,
+        };
+        if (auth.apiKey)
+            options.apiKey = auth.apiKey;
+        if (auth.headers)
+            options.headers = auth.headers;
+        const result = await complete(model, {
+            systemPrompt: VISUAL_SYSTEM_PROMPT,
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
+                        },
+                        { type: 'image', data: imageBase64, mimeType },
+                    ],
+                    timestamp: Date.now(),
+                },
+            ],
+        }, options);
+        return result.content
+            .filter((c) => c.type === 'text')
+            .map((c) => c.text)
+            .join('');
+    };
+}
+//# sourceMappingURL=vision.js.map

package/dist/vision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision.js","sourceRoot":"","sources":["../src/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAIpF,MAAM,CAAC,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;+DAc2B,CAAC;AAQhE,MAAM,UAAU,oBAAoB,CAClC,YAA+B,EAC/B,GAAqB;IAErB,OAAO,KAAK,EAAE,WAAmB,EAAE,WAAmB,EAAE,QAAgB,EAAmB,EAAE;QAC3F,MAAM,KAAK,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAChF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,iBAAiB,YAAY,CAAC,QAAQ,IAAI,YAAY,CAAC,KAAK,gCAAgC,CAC7F,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAChE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,OAAO,GAA4B;YACvC,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,IAAI;SAChB,CAAC;QACF,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC9C,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC3B,KAAK,EACL;YACE,YAAY,EAAE,oBAAoB;YAClC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAe;oBACrB,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,wEAAwE,WAAW,EAAE;yBAC5F;wBACD,EAAE,IAAI,EAAE,OAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE;qBACxD;oBACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB;aACF;SACF,EACD,OAAO,CACR,CAAC;QAEF,OAAO,MAAM,CAAC,OAAO;aAClB,MAAM,CAAC,CAAC,CAAC,EAAsB,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;aACpD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,CAAC,CAAC;AACJ,CAAC"}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@amaster.ai/pi-computer-use",
+  "version": "0.1.1-beta.0",
+  "description": "Pi extension for CUA computer-use: spawns and controls cua-computer-server",
+  "license": "Apache-2.0",
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": {
+      "default": "./package.json"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/TGYD-helige/pi.git",
+    "directory": "packages/pi-computer-use"
+  },
+  "dependencies": {
+    "ws": "^8.18.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": ">=0.74.0",
+    "@earendil-works/pi-coding-agent": ">=0.74.0",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-ai": "0.74.0",
+    "@earendil-works/pi-coding-agent": "0.74.0",
+    "@types/ws": "^8.5.0",
+    "tsup": "^8.4.0",
+    "typebox": "*",
+    "vitest": "^4.0.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc -b --pretty false",
+    "test": "vitest run src"
+  }
+}