@vitalops/opendesk-sdk 0.1.0

package/dist/tools/mouse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mouse.js","sourceRoot":"","sources":["../../src/tools/mouse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAA2B,eAAe,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,OAAO,SAAU,SAAQ,IAAI;IACjC,IAAI,GAAG,OAAO,CAAC;IACf,WAAW,GACT,4EAA4E;QAC5E,wFAAwF,CAAC;IAE3F,MAAM,GAAG;QACP,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,UAAU,EAAE;YACV,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,iBAAiB,CAAC;aAC5F;YACD,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACtB,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACtB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACzB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;YACpE,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE;SACxC;KACF,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC,GAAgB,EAAE,MAA+B;QAC7D,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,GAAG,MAG3D,CAAC;QAEF,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;QAEnF,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC7E,OAAO,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,8CAA8C,CAAC,CAAC;QAClF,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAEnF,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,MAAM;oBACT,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAE,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC;oBAChD,MAAM;gBACR,KAAK,OAAO;oBACV,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAE,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC;oBAChD,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,MAAM;gBACR,KAAK,cAAc;oBACjB,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAE,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC;oBAChD,MAAM,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACrC,MAAM;gBACR,KAAK,aAAa;oBAChB,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAE,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC;oBAChD,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAChC,MAAM;gBACR,KAAK,QAAQ;oBACX,IAAI,SAAS,KAAK,MAAM;wBAAE,MAAM,KAAK,CAAC,UAAU,CAAC,MAAgB,CAAC,CAAC;yBAC9D,IAAI,SAAS,KAAK,IAAI;wBAAE,MAAM,KAAK,CAAC,QAAQ,CAAC,MAAgB,CAAC,CAAC;yBAC/D,IAAI,SAAS,KAAK,MAAM;wBAAE,MAAM,KAAK,CAAC,UAAU,CAAC,MAAgB,CAAC,CAAC;;wBACnE,MAAM,KAAK,CAAC,WAAW,CAAC,MAAgB,CAAC,CAAC;oBAC/C,MAAM;gBACR,KAAK,MAAM;oBACT,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAK,EAAE,IAAK,CAAC,CAAC,CAAC,CAAC;oBACtD,MAAM;gBACR,KAAK,iBAAiB,CAAC,CAAC,CAAC;oBACvB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;oBACtC,OAAO,CAAC,YAAY,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;oBACvE,OAAO,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,KAAK,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9D,CAAC;gBACD;oBACE,OAAO,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,mBAAmB,MAAM,EAAE,CAAC,CAAC;YAChE,CAAC;YAED,OAAO,CAAC,YAAY,CAAC,SAAS,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,MAAM,EAAE,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAClF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}

package/dist/tools/ocr.d.ts

@@ -0,0 +1,19 @@
+import { Tool, ToolContext, ToolResult } from "./base.js";
+export declare class OCRTool extends Tool {
+    name: string;
+    description: string;
+    schema: {
+        type: string;
+        properties: {
+            region: {
+                type: string;
+                items: {
+                    type: string;
+                };
+                description: string;
+            };
+        };
+    };
+    execute(ctx: ToolContext, params: Record<string, unknown>): Promise<ToolResult>;
+}
+//# sourceMappingURL=ocr.d.ts.map

package/dist/tools/ocr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr.d.ts","sourceRoot":"","sources":["../../src/tools/ocr.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAmB,MAAM,WAAW,CAAC;AAG3E,qBAAa,OAAQ,SAAQ,IAAI;IAC/B,IAAI,SAAS;IACb,WAAW,SAAgE;IAE3E,MAAM;;;;;;;;;;;MAKJ;IAEI,OAAO,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;CA8BtF"}

package/dist/tools/ocr.js

@@ -0,0 +1,39 @@
+import { Tool, checkPermission } from "./base.js";
+import { getSandbox } from "../computer/sandbox.js";
+export class OCRTool extends Tool {
+    name = "ocr";
+    description = "Extract text from the screen or a screen region using OCR.";
+    schema = {
+        type: "object",
+        properties: {
+            region: { type: "array", items: { type: "integer" }, description: "[x, y, width, height] — omit for full screen" },
+        },
+    };
+    async execute(ctx, params) {
+        await checkPermission(ctx, "ocr", "extract text", "Run OCR on screen");
+        try {
+            const screenshot = await import("screenshot-desktop");
+            const { createWorker } = await import("tesseract.js");
+            let imgBuffer = await screenshot.default({ format: "png" });
+            // Crop to region if specified
+            if (params.region && Array.isArray(params.region) && params.region.length === 4) {
+                const { default: Jimp } = await import("jimp").catch(() => ({ default: null }));
+                if (Jimp) {
+                    const [x, y, w, h] = params.region;
+                    const img = await Jimp.read(imgBuffer);
+                    img.crop(x, y, w, h);
+                    imgBuffer = await img.getBufferAsync("image/png");
+                }
+            }
+            const worker = await createWorker("eng");
+            const { data: { text } } = await worker.recognize(imgBuffer);
+            await worker.terminate();
+            getSandbox(ctx.sessionId).recordAction("ocr", params, `${text.trim().length} chars`);
+            return this.ok("OCR", text.trim() || "(no text detected)");
+        }
+        catch (e) {
+            return this.err("OCR error", `${e}`);
+        }
+    }
+}
+//# sourceMappingURL=ocr.js.map

package/dist/tools/ocr.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr.js","sourceRoot":"","sources":["../../src/tools/ocr.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAA2B,eAAe,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,OAAO,OAAQ,SAAQ,IAAI;IAC/B,IAAI,GAAG,KAAK,CAAC;IACb,WAAW,GAAG,4DAA4D,CAAC;IAE3E,MAAM,GAAG;QACP,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,8CAA8C,EAAE;SACnH;KACF,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC,GAAgB,EAAE,MAA+B;QAC7D,MAAM,eAAe,CAAC,GAAG,EAAE,KAAK,EAAE,cAAc,EAAE,mBAAmB,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACtD,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;YAEtD,IAAI,SAAS,GAAW,MAAM,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAEpE,8BAA8B;YAC9B,IAAI,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChF,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAChF,IAAI,IAAI,EAAE,CAAC;oBACT,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAkB,CAAC;oBAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACvC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;oBACrB,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBACpD,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAC7D,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAEzB,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,QAAQ,CAAC,CAAC;YACrF,OAAO,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,oBAAoB,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;CACF"}

package/dist/tools/screenshot.d.ts

@@ -0,0 +1,28 @@
+import { Tool, ToolContext, ToolResult } from "./base.js";
+export declare class ScreenshotTool extends Tool {
+    name: string;
+    description: string;
+    schema: {
+        type: string;
+        properties: {
+            marks: {
+                type: string;
+                description: string;
+                default: boolean;
+            };
+            savePath: {
+                type: string;
+                description: string;
+            };
+            region: {
+                type: string;
+                items: {
+                    type: string;
+                };
+                description: string;
+            };
+        };
+    };
+    execute(ctx: ToolContext, params: Record<string, unknown>): Promise<ToolResult>;
+}
+//# sourceMappingURL=screenshot.d.ts.map

package/dist/tools/screenshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAmB,MAAM,WAAW,CAAC;AAG3E,qBAAa,cAAe,SAAQ,IAAI;IACtC,IAAI,SAAgB;IACpB,WAAW,SAE2E;IAEtF,MAAM;;;;;;;;;;;;;;;;;;;;MAOJ;IAEI,OAAO,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;CA+BtF"}

package/dist/tools/screenshot.js

@@ -0,0 +1,41 @@
+import { Tool, checkPermission } from "./base.js";
+import { getSandbox } from "../computer/sandbox.js";
+export class ScreenshotTool extends Tool {
+    name = "screenshot";
+    description = "Capture a screenshot of the current screen or a sub-region. " +
+        "Pass marks=true to overlay numbered boxes on interactive elements (Set-of-Marks).";
+    schema = {
+        type: "object",
+        properties: {
+            marks: { type: "boolean", description: "Overlay Set-of-Marks on interactive elements", default: false },
+            savePath: { type: "string", description: "Absolute path to save the PNG on disk" },
+            region: { type: "array", items: { type: "integer" }, description: "[x, y, width, height]" },
+        },
+    };
+    async execute(ctx, params) {
+        await checkPermission(ctx, "screenshot", "capture screen", "Take a screenshot");
+        try {
+            const screenshot = await import("screenshot-desktop");
+            const { default: capture } = screenshot;
+            let imgBuffer = await capture({ format: "png" });
+            const sandbox = getSandbox(ctx.sessionId);
+            if (params.savePath && typeof params.savePath === "string") {
+                const { writeFileSync } = await import("node:fs");
+                writeFileSync(params.savePath, imgBuffer);
+            }
+            sandbox.lastScreenshot = imgBuffer;
+            sandbox.recordAction("screenshot", params, `captured`);
+            return {
+                title: "Screenshot",
+                output: "Screenshot captured. Pass image_width and image_height from metadata to the mouse tool for correct scaling.",
+                error: false,
+                attachments: [{ filename: "screenshot.png", mediaType: "image/png", content: imgBuffer }],
+                metadata: {},
+            };
+        }
+        catch (e) {
+            return this.err("Screenshot error", `Failed to capture screenshot: ${e}`);
+        }
+    }
+}
+//# sourceMappingURL=screenshot.js.map

package/dist/tools/screenshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAA2B,eAAe,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,OAAO,cAAe,SAAQ,IAAI;IACtC,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GACT,8DAA8D;QAC9D,mFAAmF,CAAC;IAEtF,MAAM,GAAG;QACP,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,8CAA8C,EAAE,OAAO,EAAE,KAAK,EAAE;YACvG,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uCAAuC,EAAE;YAClF,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,uBAAuB,EAAE;SAC5F;KACF,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC,GAAgB,EAAE,MAA+B;QAC7D,MAAM,eAAe,CAAC,GAAG,EAAE,YAAY,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,CAAC;QAEhF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACtD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC;YAExC,IAAI,SAAS,GAAW,MAAM,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAEzD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAE1C,IAAI,MAAM,CAAC,QAAQ,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC3D,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;gBAClD,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAC5C,CAAC;YAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;YACnC,OAAO,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAEvD,OAAO;gBACL,KAAK,EAAE,YAAY;gBACnB,MAAM,EACJ,6GAA6G;gBAC/G,KAAK,EAAE,KAAK;gBACZ,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,gBAAgB,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;gBACzF,QAAQ,EAAE,EAAE;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,iCAAiC,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;CACF"}

package/dist/tools/ui.d.ts

@@ -0,0 +1,55 @@
+import { Tool, ToolContext, ToolResult } from "./base.js";
+export declare class UITool extends Tool {
+    name: string;
+    description: string;
+    schema: {
+        type: string;
+        required: string[];
+        properties: {
+            action: {
+                type: string;
+                enum: string[];
+            };
+            app: {
+                type: string;
+                description: string;
+            };
+            title: {
+                type: string;
+                description: string;
+            };
+            role: {
+                type: string;
+                description: string;
+            };
+            menu: {
+                type: string;
+                description: string;
+            };
+            menuItem: {
+                type: string;
+                description: string;
+            };
+            text: {
+                type: string;
+                description: string;
+            };
+            key: {
+                type: string;
+                description: string;
+            };
+            modifiers: {
+                type: string;
+                items: {
+                    type: string;
+                };
+                description: string;
+            };
+        };
+    };
+    execute(ctx: ToolContext, params: Record<string, unknown>): Promise<ToolResult>;
+    private executeMac;
+    private executeWin;
+    private executeLinux;
+}
+//# sourceMappingURL=ui.d.ts.map

package/dist/tools/ui.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui.d.ts","sourceRoot":"","sources":["../../src/tools/ui.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAmB,MAAM,WAAW,CAAC;AAK3E,qBAAa,MAAO,SAAQ,IAAI;IAC9B,IAAI,SAAQ;IACZ,WAAW,SAEsD;IAEjE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAcJ;IAEI,OAAO,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;YAuBvE,UAAU;YA0FV,UAAU;YA+BV,YAAY;CAqB3B"}

package/dist/tools/ui.js

@@ -0,0 +1,173 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { Tool, checkPermission } from "./base.js";
+import { getSandbox } from "../computer/sandbox.js";
+const exec = promisify(execFile);
+export class UITool extends Tool {
+    name = "ui";
+    description = "Interact with UI elements by their accessible label — the primary interaction tool. " +
+        "No pixel coordinates needed. Use this before the mouse tool.";
+    schema = {
+        type: "object",
+        required: ["action", "app"],
+        properties: {
+            action: { type: "string", enum: ["get_tree", "click", "click_menu", "type", "press_key", "get_value"] },
+            app: { type: "string", description: "Target application name" },
+            title: { type: "string", description: "Element label — for click/get_value" },
+            role: { type: "string", description: "Element role — for click/get_value" },
+            menu: { type: "string", description: "Menu bar label — for click_menu" },
+            menuItem: { type: "string", description: "Menu item label — for click_menu" },
+            text: { type: "string", description: "Text to type — for type" },
+            key: { type: "string", description: "Key name — for press_key" },
+            modifiers: { type: "array", items: { type: "string" }, description: "Modifier keys — for press_key" },
+        },
+    };
+    async execute(ctx, params) {
+        const { action, app, title, role, menu, menuItem, text, key, modifiers = [] } = params;
+        await checkPermission(ctx, "ui", `${action} in ${app}`, `UI ${action} in ${app}`);
+        const platform = process.platform;
+        try {
+            if (platform === "darwin") {
+                return await this.executeMac({ action, app, title, role, menu, menuItem, text, key, modifiers }, ctx);
+            }
+            else if (platform === "win32") {
+                return await this.executeWin({ action, app, title, text, key }, ctx);
+            }
+            else {
+                return await this.executeLinux({ action, app, title, text, key }, ctx);
+            }
+        }
+        catch (e) {
+            return this.err("UI error", `${e}`);
+        }
+    }
+    async executeMac(p, ctx) {
+        const sandbox = getSandbox(ctx.sessionId);
+        switch (p.action) {
+            case "get_tree": {
+                const script = `
+          tell application "System Events"
+            tell process "${p.app}"
+              set allElements to every UI element of window 1
+              set result to {}
+              repeat with el in allElements
+                try
+                  set result to result & {name of el & " [" & role of el & "]"}
+                end try
+              end repeat
+              return result
+            end tell
+          end tell`;
+                const { stdout } = await exec("osascript", ["-e", script]);
+                sandbox.recordAction("ui_get_tree", { app: p.app }, "ok");
+                return this.ok("UI tree", stdout.trim());
+            }
+            case "click": {
+                const by = p.title ? `whose name is "${p.title}"` : p.role ? `whose role is "${p.role}"` : "";
+                const script = `
+          tell application "System Events"
+            tell process "${p.app}"
+              click (first UI element of window 1 ${by})
+            end tell
+          end tell`;
+                await exec("osascript", ["-e", script]);
+                sandbox.recordAction("ui_click", { app: p.app, title: p.title }, "ok");
+                return this.ok("UI click", `Clicked '${p.title ?? p.role}' in ${p.app}.`);
+            }
+            case "click_menu": {
+                const script = `
+          tell application "System Events"
+            tell process "${p.app}"
+              click menu item "${p.menuItem}" of menu "${p.menu}" of menu bar 1
+            end tell
+          end tell`;
+                await exec("osascript", ["-e", script]);
+                sandbox.recordAction("ui_click_menu", { app: p.app, menu: p.menu, menuItem: p.menuItem }, "ok");
+                return this.ok("UI click menu", `Clicked ${p.app} > ${p.menu} > ${p.menuItem}.`);
+            }
+            case "type": {
+                const script = `
+          tell application "System Events"
+            tell process "${p.app}"
+              keystroke "${p.text?.replace(/"/g, '\\"')}"
+            end tell
+          end tell`;
+                await exec("osascript", ["-e", script]);
+                sandbox.recordAction("ui_type", { app: p.app }, "ok");
+                return this.ok("UI type", `Typed text in ${p.app}.`);
+            }
+            case "press_key": {
+                const modMap = { command: "command down", shift: "shift down", option: "option down", control: "control down" };
+                const using = (p.modifiers ?? []).map((m) => modMap[m] ?? m).join(", ");
+                const usingClause = using ? ` using {${using}}` : "";
+                const script = `
+          tell application "System Events"
+            tell process "${p.app}"
+              keystroke "${p.key}"${usingClause}
+            end tell
+          end tell`;
+                await exec("osascript", ["-e", script]);
+                sandbox.recordAction("ui_press_key", { app: p.app, key: p.key }, "ok");
+                return this.ok("UI press key", `Pressed ${p.key} in ${p.app}.`);
+            }
+            case "get_value": {
+                const by = p.title ? `whose name is "${p.title}"` : p.role ? `whose role is "${p.role}"` : "";
+                const script = `
+          tell application "System Events"
+            tell process "${p.app}"
+              get value of (first UI element of window 1 ${by})
+            end tell
+          end tell`;
+                const { stdout } = await exec("osascript", ["-e", script]);
+                sandbox.recordAction("ui_get_value", { app: p.app, title: p.title }, stdout.trim());
+                return this.ok("UI get value", stdout.trim());
+            }
+            default:
+                return this.err("UI error", `Unknown action: ${p.action}`);
+        }
+    }
+    async executeWin(p, ctx) {
+        const script = (ps) => exec("powershell", ["-Command", ps]);
+        const sandbox = getSandbox(ctx.sessionId);
+        switch (p.action) {
+            case "click": {
+                const ps = `
+          Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes
+          $desktop = [System.Windows.Automation.AutomationElement]::RootElement
+          $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, "${p.title}")
+          $el = $desktop.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond)
+          $invokePattern = $el.GetCurrentPattern([System.Windows.Automation.InvokePattern]::Pattern)
+          $invokePattern.Invoke()`;
+                await script(ps);
+                sandbox.recordAction("ui_click", { app: p.app, title: p.title }, "ok");
+                return this.ok("UI click", `Clicked '${p.title}' in ${p.app}.`);
+            }
+            case "type": {
+                const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${p.text}')`;
+                await script(ps);
+                sandbox.recordAction("ui_type", { app: p.app }, "ok");
+                return this.ok("UI type", `Typed text in ${p.app}.`);
+            }
+            default:
+                return this.err("UI error", `Action '${p.action}' not yet supported on Windows.`);
+        }
+    }
+    async executeLinux(p, ctx) {
+        const sandbox = getSandbox(ctx.sessionId);
+        switch (p.action) {
+            case "click": {
+                await exec("xdotool", ["search", "--name", p.title ?? p.app, "click", "1"]);
+                sandbox.recordAction("ui_click", { app: p.app, title: p.title }, "ok");
+                return this.ok("UI click", `Clicked '${p.title}' in ${p.app}.`);
+            }
+            case "type": {
+                await exec("xdotool", ["type", "--clearmodifiers", p.text ?? ""]);
+                sandbox.recordAction("ui_type", { app: p.app }, "ok");
+                return this.ok("UI type", `Typed text in ${p.app}.`);
+            }
+            default:
+                return this.err("UI error", `Action '${p.action}' not yet supported on Linux.`);
+        }
+    }
+}
+//# sourceMappingURL=ui.js.map

package/dist/tools/ui.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui.js","sourceRoot":"","sources":["../../src/tools/ui.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,IAAI,EAA2B,eAAe,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAEjC,MAAM,OAAO,MAAO,SAAQ,IAAI;IAC9B,IAAI,GAAG,IAAI,CAAC;IACZ,WAAW,GACT,sFAAsF;QACtF,8DAA8D,CAAC;IAEjE,MAAM,GAAG;QACP,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC;QAC3B,UAAU,EAAE;YACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,EAAE;YACvG,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,yBAAyB,EAAE;YAC/D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;YAC7E,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oCAAoC,EAAE;YAC3E,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iCAAiC,EAAE;YACxE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kCAAkC,EAAE;YAC7E,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,yBAAyB,EAAE;YAChE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0BAA0B,EAAE;YAChE,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACtG;KACF,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC,GAAgB,EAAE,MAA+B;QAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,MAG/E,CAAC;QAEF,MAAM,eAAe,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,GAAG,EAAE,EAAE,MAAM,MAAM,OAAO,GAAG,EAAE,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAElC,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC1B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,CAAC;YACxG,CAAC;iBAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACvE,CAAC;iBAAM,CAAC;gBACN,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,CAAsJ,EACtJ,GAAgB;QAEhB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAE1C,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;YACjB,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,MAAM,GAAG;;4BAEK,CAAC,CAAC,GAAG;;;;;;;;;;mBAUd,CAAC;gBACZ,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC3D,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC1D,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3C,CAAC;YACD,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9F,MAAM,MAAM,GAAG;;4BAEK,CAAC,CAAC,GAAG;oDACmB,EAAE;;mBAEnC,CAAC;gBACZ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBACxC,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAC5E,CAAC;YACD,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,MAAM,GAAG;;4BAEK,CAAC,CAAC,GAAG;iCACA,CAAC,CAAC,QAAQ,cAAc,CAAC,CAAC,IAAI;;mBAE5C,CAAC;gBACZ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBACxC,OAAO,CAAC,YAAY,CAAC,eAAe,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,CAAC;gBAChG,OAAO,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;YACnF,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,MAAM,GAAG;;4BAEK,CAAC,CAAC,GAAG;2BACN,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;;mBAEpC,CAAC;gBACZ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBACxC,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBACtD,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YACvD,CAAC;YACD,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,MAAM,MAAM,GAA2B,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;gBACxI,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxE,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,MAAM,MAAM,GAAG;;4BAEK,CAAC,CAAC,GAAG;2BACN,CAAC,CAAC,GAAG,IAAI,WAAW;;mBAE5B,CAAC;gBACZ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBACxC,OAAO,CAAC,YAAY,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9F,MAAM,MAAM,GAAG;;4BAEK,CAAC,CAAC,GAAG;2DAC0B,EAAE;;mBAE1C,CAAC;gBACZ,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC3D,OAAO,CAAC,YAAY,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpF,OAAO,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,CAAC;YACD;gBACE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,CAA+E,EAC/E,GAAgB;QAEhB,MAAM,MAAM,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAE1C,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;YACjB,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,EAAE,GAAG;;;yIAGsH,CAAC,CAAC,KAAK;;;kCAG9G,CAAC;gBAC3B,MAAM,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjB,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,EAAE,GAAG,2FAA2F,CAAC,CAAC,IAAI,IAAI,CAAC;gBACjH,MAAM,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjB,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBACtD,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YACvD,CAAC;YACD;gBACE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,MAAM,iCAAiC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,CAA+E,EAC/E,GAAgB;QAEhB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAE1C,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;YACjB,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC5E,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAClE,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;gBAClE,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBACtD,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YACvD,CAAC;YACD;gBACE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,MAAM,+BAA+B,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;CACF"}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@vitalops/opendesk-sdk",
+  "version": "0.1.0",
+  "description": "JavaScript/TypeScript SDK for opendesk — give any AI agent eyes and hands on your desktop",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "bin": {
+    "opendesk-js": "bin/opendesk.js",
+    "opendesk-js-mcp": "bin/opendesk-mcp.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "typecheck": "tsc --noEmit"
+  },
+  "files": ["dist", "bin", "README.md"],
+  "keywords": ["opendesk", "computer-use", "desktop-automation", "mcp", "ai-agent"],
+  "license": "MIT",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "@nut-tree-fork/nut-js": "^4.2.0",
+    "clipboardy": "^4.0.0",
+    "node-cron": "^3.0.0",
+    "screenshot-desktop": "^1.15.0",
+    "tesseract.js": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "@types/node-cron": "^3.0.0",
+    "typescript": "^5.4.0"
+  },
+  "engines": { "node": ">=18.0.0" },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vitalops/opendesk.git",
+    "directory": "js"
+  }
+}