@elizaos/plugin-computeruse 2.0.0-alpha.6 → 2.0.0-alpha.8

package/dist/actions/click-coordinates.d.ts

@@ -0,0 +1,3 @@
+import type { Action } from "@elizaos/core";
+export declare const computeruseClickCoordinatesAction: Action;
+//# sourceMappingURL=click-coordinates.d.ts.map

package/dist/actions/click-coordinates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"click-coordinates.d.ts","sourceRoot":"","sources":["../../src/actions/click-coordinates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAQP,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,iCAAiC,EAAE,MA0F/C,CAAC"}

package/dist/actions/click-coordinates.js

@@ -0,0 +1,81 @@
+import { logger } from "@elizaos/core";
+import { getNumberParam, getStringParam } from "../utils/params.js";
+export const computeruseClickCoordinatesAction = {
+    name: "COMPUTERUSE_CLICK_COORDINATES",
+    description: "Clicks at specific screen coordinates (x, y). Use for precise clicking at known positions.",
+    similes: ["CLICK_AT", "CLICK_POSITION", "CLICK_XY"],
+    parameters: [
+        {
+            name: "x",
+            description: "X coordinate to click at",
+            required: true,
+            schema: { type: "number" },
+            examples: [500, 960],
+        },
+        {
+            name: "y",
+            description: "Y coordinate to click at",
+            required: true,
+            schema: { type: "number" },
+            examples: [300, 540],
+        },
+        {
+            name: "button",
+            description: "Mouse button: 'left', 'right', or 'middle'. Default: 'left'",
+            required: false,
+            schema: { type: "string" },
+            examples: ["left", "right"],
+        },
+        {
+            name: "numClicks",
+            description: "Number of clicks (1=single, 2=double). Default: 1",
+            required: false,
+            schema: { type: "number", default: 1, minimum: 1, maximum: 3 },
+            examples: [1, 2],
+        },
+    ],
+    validate: async (runtime, message) => {
+        const service = runtime.getService("computeruse");
+        return !!service && service.isEnabled();
+    },
+    handler: async (runtime, message, _state, options, callback) => {
+        const service = runtime.getService("computeruse");
+        if (!service) {
+            return { success: false, text: "ComputerUse service not available" };
+        }
+        const params = options?.parameters;
+        const x = getNumberParam(params, "x");
+        const y = getNumberParam(params, "y");
+        const button = getStringParam(params, "button") ?? "left";
+        const numClicks = getNumberParam(params, "numClicks") ?? 1;
+        if (x === undefined || y === undefined) {
+            return { success: false, text: "Missing required parameters: x and y" };
+        }
+        const backend = service.getBackendName();
+        try {
+            await service.clickCoordinates(x, y, button, numClicks);
+            const response = {
+                text: `Clicked at coordinates (${x}, ${y}) with ${button} button`,
+                actions: ["COMPUTERUSE_CLICK_COORDINATES"],
+                source: message.content?.source ?? "action",
+            };
+            await callback?.(response);
+            return {
+                success: true,
+                text: response.text ?? "",
+                values: { x, y, button, numClicks },
+                data: { x, y, button, numClicks, backend },
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error(`[computeruse] click_coordinates failed: ${msg}`);
+            return {
+                success: false,
+                text: `ComputerUse click at coordinates failed: ${msg}`,
+                values: { x, y, button, numClicks },
+            };
+        }
+    },
+};
+//# sourceMappingURL=click-coordinates.js.map

package/dist/actions/click-coordinates.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click-coordinates.js","sourceRoot":"","sources":["../../src/actions/click-coordinates.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpE,MAAM,CAAC,MAAM,iCAAiC,GAAW;IACvD,IAAI,EAAE,+BAA+B;IACrC,WAAW,EACT,4FAA4F;IAC9F,OAAO,EAAE,CAAC,UAAU,EAAE,gBAAgB,EAAE,UAAU,CAAC;IACnD,UAAU,EAAE;QACV;YACE,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,0BAA0B;YACvC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;SACrB;QACD;YACE,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,0BAA0B;YACvC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;SACrB;QACD;YACE,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,6DAA6D;YAC1E,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;SAC5B;QACD;YACE,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,mDAAmD;YAChE,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;YAC9D,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;SACjB;KACF;IAED,QAAQ,EAAE,KAAK,EAAE,OAAsB,EAAE,OAAe,EAAoB,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EACZ,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACH,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC;QAC1D,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,sCAAsC,EAAE,CAAC;QAC1E,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;YAExD,MAAM,QAAQ,GAAY;gBACxB,IAAI,EAAE,2BAA2B,CAAC,KAAK,CAAC,UAAU,MAAM,SAAS;gBACjE,OAAO,EAAE,CAAC,+BAA+B,CAAC;gBAC1C,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC5C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;gBACnC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE;aAC3C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;YAC/D,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,4CAA4C,GAAG,EAAE;gBACvD,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC","sourcesContent":["import type {\n  Action,\n  ActionResult,\n  Content,\n  HandlerCallback,\n  HandlerOptions,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getNumberParam, getStringParam } from \"../utils/params.js\";\n\nexport const computeruseClickCoordinatesAction: Action = {\n  name: \"COMPUTERUSE_CLICK_COORDINATES\",\n  description:\n    \"Clicks at specific screen coordinates (x, y). Use for precise clicking at known positions.\",\n  similes: [\"CLICK_AT\", \"CLICK_POSITION\", \"CLICK_XY\"],\n  parameters: [\n    {\n      name: \"x\",\n      description: \"X coordinate to click at\",\n      required: true,\n      schema: { type: \"number\" },\n      examples: [500, 960],\n    },\n    {\n      name: \"y\",\n      description: \"Y coordinate to click at\",\n      required: true,\n      schema: { type: \"number\" },\n      examples: [300, 540],\n    },\n    {\n      name: \"button\",\n      description: \"Mouse button: 'left', 'right', or 'middle'. Default: 'left'\",\n      required: false,\n      schema: { type: \"string\" },\n      examples: [\"left\", \"right\"],\n    },\n    {\n      name: \"numClicks\",\n      description: \"Number of clicks (1=single, 2=double). Default: 1\",\n      required: false,\n      schema: { type: \"number\", default: 1, minimum: 1, maximum: 3 },\n      examples: [1, 2],\n    },\n  ],\n\n  validate: async (runtime: IAgentRuntime, message: Memory): Promise<boolean> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    return !!service && service.isEnabled();\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state?: State,\n    options?: HandlerOptions,\n    callback?: HandlerCallback\n  ): Promise<ActionResult> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    if (!service) {\n      return { success: false, text: \"ComputerUse service not available\" };\n    }\n\n    const params = options?.parameters;\n    const x = getNumberParam(params, \"x\");\n    const y = getNumberParam(params, \"y\");\n    const button = getStringParam(params, \"button\") ?? \"left\";\n    const numClicks = getNumberParam(params, \"numClicks\") ?? 1;\n\n    if (x === undefined || y === undefined) {\n      return { success: false, text: \"Missing required parameters: x and y\" };\n    }\n\n    const backend = service.getBackendName();\n\n    try {\n      await service.clickCoordinates(x, y, button, numClicks);\n\n      const response: Content = {\n        text: `Clicked at coordinates (${x}, ${y}) with ${button} button`,\n        actions: [\"COMPUTERUSE_CLICK_COORDINATES\"],\n        source: message.content?.source ?? \"action\",\n      };\n      await callback?.(response);\n      return {\n        success: true,\n        text: response.text ?? \"\",\n        values: { x, y, button, numClicks },\n        data: { x, y, button, numClicks, backend },\n      };\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.error(`[computeruse] click_coordinates failed: ${msg}`);\n      return {\n        success: false,\n        text: `ComputerUse click at coordinates failed: ${msg}`,\n        values: { x, y, button, numClicks },\n      };\n    }\n  },\n};\n"]}

package/dist/actions/drag.d.ts

@@ -0,0 +1,3 @@
+import type { Action } from "@elizaos/core";
+export declare const computeruseDragAction: Action;
+//# sourceMappingURL=drag.d.ts.map

package/dist/actions/drag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"drag.d.ts","sourceRoot":"","sources":["../../src/actions/drag.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAQP,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,qBAAqB,EAAE,MAkGnC,CAAC"}

package/dist/actions/drag.js

@@ -0,0 +1,87 @@
+import { logger } from "@elizaos/core";
+import { getNumberParam } from "../utils/params.js";
+export const computeruseDragAction = {
+    name: "COMPUTERUSE_DRAG",
+    description: "Drag the mouse from one position to another. Use for drag-and-drop, resizing, selecting, sliders.",
+    similes: ["DRAG", "DRAG_AND_DROP", "MOUSE_DRAG"],
+    parameters: [
+        {
+            name: "startX",
+            description: "Starting X coordinate",
+            required: true,
+            schema: { type: "number" },
+            examples: [100],
+        },
+        {
+            name: "startY",
+            description: "Starting Y coordinate",
+            required: true,
+            schema: { type: "number" },
+            examples: [200],
+        },
+        {
+            name: "endX",
+            description: "Ending X coordinate",
+            required: true,
+            schema: { type: "number" },
+            examples: [500],
+        },
+        {
+            name: "endY",
+            description: "Ending Y coordinate",
+            required: true,
+            schema: { type: "number" },
+            examples: [200],
+        },
+    ],
+    validate: async (runtime, message) => {
+        const service = runtime.getService("computeruse");
+        return !!service && service.isEnabled();
+    },
+    handler: async (runtime, message, _state, options, callback) => {
+        const service = runtime.getService("computeruse");
+        if (!service) {
+            return { success: false, text: "ComputerUse service not available" };
+        }
+        const params = options?.parameters;
+        const startX = getNumberParam(params, "startX");
+        const startY = getNumberParam(params, "startY");
+        const endX = getNumberParam(params, "endX");
+        const endY = getNumberParam(params, "endY");
+        if (startX === undefined ||
+            startY === undefined ||
+            endX === undefined ||
+            endY === undefined) {
+            return {
+                success: false,
+                text: "Missing required parameters: startX, startY, endX, endY",
+            };
+        }
+        const backend = service.getBackendName();
+        try {
+            await service.drag(startX, startY, endX, endY);
+            const response = {
+                text: `Dragged from (${startX}, ${startY}) to (${endX}, ${endY})`,
+                actions: ["COMPUTERUSE_DRAG"],
+                source: message.content?.source ?? "action",
+            };
+            await callback?.(response);
+            return {
+                success: true,
+                text: response.text ?? "",
+                values: { startX, startY, endX, endY },
+                data: { startX, startY, endX, endY, backend },
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error(`[computeruse] drag failed: ${msg}`);
+            return {
+                success: false,
+                text: `ComputerUse drag failed: ${msg}`,
+                values: { startX, startY, endX, endY },
+            };
+        }
+    },
+};
+//# sourceMappingURL=drag.js.map

package/dist/actions/drag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"drag.js","sourceRoot":"","sources":["../../src/actions/drag.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,CAAC,MAAM,qBAAqB,GAAW;IAC3C,IAAI,EAAE,kBAAkB;IACxB,WAAW,EACT,mGAAmG;IACrG,OAAO,EAAE,CAAC,MAAM,EAAE,eAAe,EAAE,YAAY,CAAC;IAChD,UAAU,EAAE;QACV;YACE,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,uBAAuB;YACpC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,CAAC;SAChB;QACD;YACE,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,uBAAuB;YACpC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,CAAC;SAChB;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,qBAAqB;YAClC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,CAAC;SAChB;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,qBAAqB;YAClC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,CAAC;SAChB;KACF;IAED,QAAQ,EAAE,KAAK,EAAE,OAAsB,EAAE,OAAe,EAAoB,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EACZ,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACH,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAE5C,IACE,MAAM,KAAK,SAAS;YACpB,MAAM,KAAK,SAAS;YACpB,IAAI,KAAK,SAAS;YAClB,IAAI,KAAK,SAAS,EAClB,CAAC;YACD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,yDAAyD;aAChE,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAE/C,MAAM,QAAQ,GAAY;gBACxB,IAAI,EAAE,iBAAiB,MAAM,KAAK,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG;gBACjE,OAAO,EAAE,CAAC,kBAAkB,CAAC;gBAC7B,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC5C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE;gBACtC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;aAC9C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,4BAA4B,GAAG,EAAE;gBACvC,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC","sourcesContent":["import type {\n  Action,\n  ActionResult,\n  Content,\n  HandlerCallback,\n  HandlerOptions,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getNumberParam } from \"../utils/params.js\";\n\nexport const computeruseDragAction: Action = {\n  name: \"COMPUTERUSE_DRAG\",\n  description:\n    \"Drag the mouse from one position to another. Use for drag-and-drop, resizing, selecting, sliders.\",\n  similes: [\"DRAG\", \"DRAG_AND_DROP\", \"MOUSE_DRAG\"],\n  parameters: [\n    {\n      name: \"startX\",\n      description: \"Starting X coordinate\",\n      required: true,\n      schema: { type: \"number\" },\n      examples: [100],\n    },\n    {\n      name: \"startY\",\n      description: \"Starting Y coordinate\",\n      required: true,\n      schema: { type: \"number\" },\n      examples: [200],\n    },\n    {\n      name: \"endX\",\n      description: \"Ending X coordinate\",\n      required: true,\n      schema: { type: \"number\" },\n      examples: [500],\n    },\n    {\n      name: \"endY\",\n      description: \"Ending Y coordinate\",\n      required: true,\n      schema: { type: \"number\" },\n      examples: [200],\n    },\n  ],\n\n  validate: async (runtime: IAgentRuntime, message: Memory): Promise<boolean> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    return !!service && service.isEnabled();\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state?: State,\n    options?: HandlerOptions,\n    callback?: HandlerCallback\n  ): Promise<ActionResult> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    if (!service) {\n      return { success: false, text: \"ComputerUse service not available\" };\n    }\n\n    const params = options?.parameters;\n    const startX = getNumberParam(params, \"startX\");\n    const startY = getNumberParam(params, \"startY\");\n    const endX = getNumberParam(params, \"endX\");\n    const endY = getNumberParam(params, \"endY\");\n\n    if (\n      startX === undefined ||\n      startY === undefined ||\n      endX === undefined ||\n      endY === undefined\n    ) {\n      return {\n        success: false,\n        text: \"Missing required parameters: startX, startY, endX, endY\",\n      };\n    }\n\n    const backend = service.getBackendName();\n\n    try {\n      await service.drag(startX, startY, endX, endY);\n\n      const response: Content = {\n        text: `Dragged from (${startX}, ${startY}) to (${endX}, ${endY})`,\n        actions: [\"COMPUTERUSE_DRAG\"],\n        source: message.content?.source ?? \"action\",\n      };\n      await callback?.(response);\n      return {\n        success: true,\n        text: response.text ?? \"\",\n        values: { startX, startY, endX, endY },\n        data: { startX, startY, endX, endY, backend },\n      };\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.error(`[computeruse] drag failed: ${msg}`);\n      return {\n        success: false,\n        text: `ComputerUse drag failed: ${msg}`,\n        values: { startX, startY, endX, endY },\n      };\n    }\n  },\n};\n"]}

package/dist/actions/press-key.d.ts

@@ -0,0 +1,3 @@
+import type { Action } from "@elizaos/core";
+export declare const computerusePressKeyAction: Action;
+//# sourceMappingURL=press-key.d.ts.map

package/dist/actions/press-key.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"press-key.d.ts","sourceRoot":"","sources":["../../src/actions/press-key.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAQP,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,yBAAyB,EAAE,MAkEvC,CAAC"}

package/dist/actions/press-key.js

@@ -0,0 +1,57 @@
+import { logger } from "@elizaos/core";
+import { getStringParam } from "../utils/params.js";
+export const computerusePressKeyAction = {
+    name: "COMPUTERUSE_PRESS_KEY",
+    description: "Press a keyboard key or key combination (hotkey). Use for shortcuts, Enter, Tab, etc.",
+    similes: ["PRESS_KEY", "HOTKEY", "KEY_PRESS", "KEYBOARD_SHORTCUT"],
+    parameters: [
+        {
+            name: "keys",
+            description: "Key or key combination to press. Examples: 'enter', 'ctrl+c', 'alt+tab', 'ctrl+shift+s'",
+            required: true,
+            schema: { type: "string" },
+            examples: ["enter", "ctrl+c", "alt+tab", "tab", "escape"],
+        },
+    ],
+    validate: async (runtime, message) => {
+        const service = runtime.getService("computeruse");
+        return !!service && service.isEnabled();
+    },
+    handler: async (runtime, message, _state, options, callback) => {
+        const service = runtime.getService("computeruse");
+        if (!service) {
+            return { success: false, text: "ComputerUse service not available" };
+        }
+        const params = options?.parameters;
+        const keys = getStringParam(params, "keys");
+        if (!keys) {
+            return { success: false, text: "Missing required parameter: keys" };
+        }
+        const backend = service.getBackendName();
+        try {
+            await service.pressKey(keys);
+            const response = {
+                text: `Pressed key: ${keys}`,
+                actions: ["COMPUTERUSE_PRESS_KEY"],
+                source: message.content?.source ?? "action",
+            };
+            await callback?.(response);
+            return {
+                success: true,
+                text: response.text ?? "",
+                values: { keys },
+                data: { keys, backend },
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error(`[computeruse] press_key failed: ${msg}`);
+            return {
+                success: false,
+                text: `ComputerUse press key failed: ${msg}`,
+                values: { keys },
+            };
+        }
+    },
+};
+//# sourceMappingURL=press-key.js.map

package/dist/actions/press-key.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"press-key.js","sourceRoot":"","sources":["../../src/actions/press-key.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,CAAC,MAAM,yBAAyB,GAAW;IAC/C,IAAI,EAAE,uBAAuB;IAC7B,WAAW,EACT,uFAAuF;IACzF,OAAO,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,CAAC;IAClE,UAAU,EAAE;QACV;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,yFAAyF;YAC3F,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC;SAC1D;KACF;IAED,QAAQ,EAAE,KAAK,EAAE,OAAsB,EAAE,OAAe,EAAoB,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EACZ,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACH,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,kCAAkC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE7B,MAAM,QAAQ,GAAY;gBACxB,IAAI,EAAE,gBAAgB,IAAI,EAAE;gBAC5B,OAAO,EAAE,CAAC,uBAAuB,CAAC;gBAClC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC5C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,IAAI,EAAE;gBAChB,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;YACvD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,iCAAiC,GAAG,EAAE;gBAC5C,MAAM,EAAE,EAAE,IAAI,EAAE;aACjB,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC","sourcesContent":["import type {\n  Action,\n  ActionResult,\n  Content,\n  HandlerCallback,\n  HandlerOptions,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getStringParam } from \"../utils/params.js\";\n\nexport const computerusePressKeyAction: Action = {\n  name: \"COMPUTERUSE_PRESS_KEY\",\n  description:\n    \"Press a keyboard key or key combination (hotkey). Use for shortcuts, Enter, Tab, etc.\",\n  similes: [\"PRESS_KEY\", \"HOTKEY\", \"KEY_PRESS\", \"KEYBOARD_SHORTCUT\"],\n  parameters: [\n    {\n      name: \"keys\",\n      description:\n        \"Key or key combination to press. Examples: 'enter', 'ctrl+c', 'alt+tab', 'ctrl+shift+s'\",\n      required: true,\n      schema: { type: \"string\" },\n      examples: [\"enter\", \"ctrl+c\", \"alt+tab\", \"tab\", \"escape\"],\n    },\n  ],\n\n  validate: async (runtime: IAgentRuntime, message: Memory): Promise<boolean> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    return !!service && service.isEnabled();\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state?: State,\n    options?: HandlerOptions,\n    callback?: HandlerCallback\n  ): Promise<ActionResult> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    if (!service) {\n      return { success: false, text: \"ComputerUse service not available\" };\n    }\n\n    const params = options?.parameters;\n    const keys = getStringParam(params, \"keys\");\n    if (!keys) {\n      return { success: false, text: \"Missing required parameter: keys\" };\n    }\n\n    const backend = service.getBackendName();\n\n    try {\n      await service.pressKey(keys);\n\n      const response: Content = {\n        text: `Pressed key: ${keys}`,\n        actions: [\"COMPUTERUSE_PRESS_KEY\"],\n        source: message.content?.source ?? \"action\",\n      };\n      await callback?.(response);\n      return {\n        success: true,\n        text: response.text ?? \"\",\n        values: { keys },\n        data: { keys, backend },\n      };\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.error(`[computeruse] press_key failed: ${msg}`);\n      return {\n        success: false,\n        text: `ComputerUse press key failed: ${msg}`,\n        values: { keys },\n      };\n    }\n  },\n};\n"]}

package/dist/actions/screenshot.d.ts

@@ -0,0 +1,3 @@
+import type { Action } from "@elizaos/core";
+export declare const computeruseScreenshotAction: Action;
+//# sourceMappingURL=screenshot.d.ts.map

package/dist/actions/screenshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/actions/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAQP,MAAM,eAAe,CAAC;AAIvB,eAAO,MAAM,2BAA2B,EAAE,MAgDzC,CAAC"}

package/dist/actions/screenshot.js

@@ -0,0 +1,41 @@
+import { logger } from "@elizaos/core";
+export const computeruseScreenshotAction = {
+    name: "COMPUTERUSE_SCREENSHOT",
+    description: "Capture a screenshot of the current screen.",
+    similes: ["SCREENSHOT", "CAPTURE_SCREEN", "TAKE_SCREENSHOT"],
+    parameters: [],
+    validate: async (runtime, message) => {
+        const service = runtime.getService("computeruse");
+        return !!service && service.isEnabled();
+    },
+    handler: async (runtime, message, _state, options, callback) => {
+        const service = runtime.getService("computeruse");
+        if (!service) {
+            return { success: false, text: "ComputerUse service not available" };
+        }
+        const backend = service.getBackendName();
+        try {
+            const screenshotData = await service.screenshot();
+            const response = {
+                text: "Screenshot captured",
+                actions: ["COMPUTERUSE_SCREENSHOT"],
+                source: message.content?.source ?? "action",
+            };
+            await callback?.(response);
+            return {
+                success: true,
+                text: response.text ?? "",
+                data: { backend, screenshotLength: screenshotData.length },
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error(`[computeruse] screenshot failed: ${msg}`);
+            return {
+                success: false,
+                text: `ComputerUse screenshot failed: ${msg}`,
+            };
+        }
+    },
+};
+//# sourceMappingURL=screenshot.js.map

package/dist/actions/screenshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/actions/screenshot.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAGvC,MAAM,CAAC,MAAM,2BAA2B,GAAW;IACjD,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,6CAA6C;IAC1D,OAAO,EAAE,CAAC,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,CAAC;IAC5D,UAAU,EAAE,EAAE;IAEd,QAAQ,EAAE,KAAK,EAAE,OAAsB,EAAE,OAAe,EAAoB,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EACZ,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACH,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAElD,MAAM,QAAQ,GAAY;gBACxB,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EAAE,CAAC,wBAAwB,CAAC;gBACnC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC5C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,IAAI,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE;aAC3D,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;YACxD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,kCAAkC,GAAG,EAAE;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC","sourcesContent":["import type {\n  Action,\n  ActionResult,\n  Content,\n  HandlerCallback,\n  HandlerOptions,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\n\nexport const computeruseScreenshotAction: Action = {\n  name: \"COMPUTERUSE_SCREENSHOT\",\n  description: \"Capture a screenshot of the current screen.\",\n  similes: [\"SCREENSHOT\", \"CAPTURE_SCREEN\", \"TAKE_SCREENSHOT\"],\n  parameters: [],\n\n  validate: async (runtime: IAgentRuntime, message: Memory): Promise<boolean> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    return !!service && service.isEnabled();\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state?: State,\n    options?: HandlerOptions,\n    callback?: HandlerCallback\n  ): Promise<ActionResult> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    if (!service) {\n      return { success: false, text: \"ComputerUse service not available\" };\n    }\n\n    const backend = service.getBackendName();\n\n    try {\n      const screenshotData = await service.screenshot();\n\n      const response: Content = {\n        text: \"Screenshot captured\",\n        actions: [\"COMPUTERUSE_SCREENSHOT\"],\n        source: message.content?.source ?? \"action\",\n      };\n      await callback?.(response);\n      return {\n        success: true,\n        text: response.text ?? \"\",\n        data: { backend, screenshotLength: screenshotData.length },\n      };\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.error(`[computeruse] screenshot failed: ${msg}`);\n      return {\n        success: false,\n        text: `ComputerUse screenshot failed: ${msg}`,\n      };\n    }\n  },\n};\n"]}

package/dist/actions/scroll.d.ts

@@ -0,0 +1,3 @@
+import type { Action } from "@elizaos/core";
+export declare const computeruseScrollAction: Action;
+//# sourceMappingURL=scroll.d.ts.map

package/dist/actions/scroll.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scroll.d.ts","sourceRoot":"","sources":["../../src/actions/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAQP,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,uBAAuB,EAAE,MAsFrC,CAAC"}

package/dist/actions/scroll.js

@@ -0,0 +1,79 @@
+import { logger } from "@elizaos/core";
+import { getNumberParam, getStringParam } from "../utils/params.js";
+export const computeruseScrollAction = {
+    name: "COMPUTERUSE_SCROLL",
+    description: "Scroll the mouse wheel at the current or specified position.",
+    similes: ["SCROLL", "SCROLL_UP", "SCROLL_DOWN", "MOUSE_SCROLL"],
+    parameters: [
+        {
+            name: "x",
+            description: "X coordinate to scroll at (optional)",
+            required: false,
+            schema: { type: "number" },
+            examples: [500],
+        },
+        {
+            name: "y",
+            description: "Y coordinate to scroll at (optional)",
+            required: false,
+            schema: { type: "number" },
+            examples: [500],
+        },
+        {
+            name: "direction",
+            description: "Scroll direction: 'up' or 'down'. Default: 'down'",
+            required: false,
+            schema: { type: "string" },
+            examples: ["up", "down"],
+        },
+        {
+            name: "amount",
+            description: "Number of scroll clicks. Default: 3",
+            required: false,
+            schema: { type: "number", default: 3, minimum: 1 },
+            examples: [3, 5, 10],
+        },
+    ],
+    validate: async (runtime, message) => {
+        const service = runtime.getService("computeruse");
+        return !!service && service.isEnabled();
+    },
+    handler: async (runtime, message, _state, options, callback) => {
+        const service = runtime.getService("computeruse");
+        if (!service) {
+            return { success: false, text: "ComputerUse service not available" };
+        }
+        const params = options?.parameters;
+        const x = getNumberParam(params, "x");
+        const y = getNumberParam(params, "y");
+        const direction = getStringParam(params, "direction") ?? "down";
+        const amount = getNumberParam(params, "amount") ?? 3;
+        const backend = service.getBackendName();
+        try {
+            await service.scroll(direction, amount, x, y);
+            const posStr = x !== undefined ? `(${x}, ${y})` : "current position";
+            const response = {
+                text: `Scrolled ${direction} by ${amount} at ${posStr}`,
+                actions: ["COMPUTERUSE_SCROLL"],
+                source: message.content?.source ?? "action",
+            };
+            await callback?.(response);
+            return {
+                success: true,
+                text: response.text ?? "",
+                values: { x, y, direction, amount },
+                data: { x, y, direction, amount, backend },
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error(`[computeruse] scroll failed: ${msg}`);
+            return {
+                success: false,
+                text: `ComputerUse scroll failed: ${msg}`,
+                values: { x, y, direction, amount },
+            };
+        }
+    },
+};
+//# sourceMappingURL=scroll.js.map

package/dist/actions/scroll.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../src/actions/scroll.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpE,MAAM,CAAC,MAAM,uBAAuB,GAAW;IAC7C,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,8DAA8D;IAC3E,OAAO,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,CAAC;IAC/D,UAAU,EAAE;QACV;YACE,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,sCAAsC;YACnD,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,CAAC;SAChB;QACD;YACE,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,sCAAsC;YACnD,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,CAAC;SAChB;QACD;YACE,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,mDAAmD;YAChE,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC;SACzB;QACD;YACE,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,qCAAqC;YAClD,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;YAClD,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;SACrB;KACF;IAED,QAAQ,EAAE,KAAK,EAAE,OAAsB,EAAE,OAAe,EAAoB,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EACZ,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACH,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC;QAChE,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAE9C,MAAM,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC;YACrE,MAAM,QAAQ,GAAY;gBACxB,IAAI,EAAE,YAAY,SAAS,OAAO,MAAM,OAAO,MAAM,EAAE;gBACvD,OAAO,EAAE,CAAC,oBAAoB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC5C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE;gBACnC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;aAC3C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,gCAAgC,GAAG,EAAE,CAAC,CAAC;YACpD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,8BAA8B,GAAG,EAAE;gBACzC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC","sourcesContent":["import type {\n  Action,\n  ActionResult,\n  Content,\n  HandlerCallback,\n  HandlerOptions,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getNumberParam, getStringParam } from \"../utils/params.js\";\n\nexport const computeruseScrollAction: Action = {\n  name: \"COMPUTERUSE_SCROLL\",\n  description: \"Scroll the mouse wheel at the current or specified position.\",\n  similes: [\"SCROLL\", \"SCROLL_UP\", \"SCROLL_DOWN\", \"MOUSE_SCROLL\"],\n  parameters: [\n    {\n      name: \"x\",\n      description: \"X coordinate to scroll at (optional)\",\n      required: false,\n      schema: { type: \"number\" },\n      examples: [500],\n    },\n    {\n      name: \"y\",\n      description: \"Y coordinate to scroll at (optional)\",\n      required: false,\n      schema: { type: \"number\" },\n      examples: [500],\n    },\n    {\n      name: \"direction\",\n      description: \"Scroll direction: 'up' or 'down'. Default: 'down'\",\n      required: false,\n      schema: { type: \"string\" },\n      examples: [\"up\", \"down\"],\n    },\n    {\n      name: \"amount\",\n      description: \"Number of scroll clicks. Default: 3\",\n      required: false,\n      schema: { type: \"number\", default: 3, minimum: 1 },\n      examples: [3, 5, 10],\n    },\n  ],\n\n  validate: async (runtime: IAgentRuntime, message: Memory): Promise<boolean> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    return !!service && service.isEnabled();\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state?: State,\n    options?: HandlerOptions,\n    callback?: HandlerCallback\n  ): Promise<ActionResult> => {\n    const service = runtime.getService<ComputerUseService>(\"computeruse\");\n    if (!service) {\n      return { success: false, text: \"ComputerUse service not available\" };\n    }\n\n    const params = options?.parameters;\n    const x = getNumberParam(params, \"x\");\n    const y = getNumberParam(params, \"y\");\n    const direction = getStringParam(params, \"direction\") ?? \"down\";\n    const amount = getNumberParam(params, \"amount\") ?? 3;\n\n    const backend = service.getBackendName();\n\n    try {\n      await service.scroll(direction, amount, x, y);\n\n      const posStr = x !== undefined ? `(${x}, ${y})` : \"current position\";\n      const response: Content = {\n        text: `Scrolled ${direction} by ${amount} at ${posStr}`,\n        actions: [\"COMPUTERUSE_SCROLL\"],\n        source: message.content?.source ?? \"action\",\n      };\n      await callback?.(response);\n      return {\n        success: true,\n        text: response.text ?? \"\",\n        values: { x, y, direction, amount },\n        data: { x, y, direction, amount, backend },\n      };\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.error(`[computeruse] scroll failed: ${msg}`);\n      return {\n        success: false,\n        text: `ComputerUse scroll failed: ${msg}`,\n        values: { x, y, direction, amount },\n      };\n    }\n  },\n};\n"]}

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,MAAM,EAAE,MAAM,eAAe,CAAC;AAG3D,OAAO,uBAAuB,CAAC;AAQ/B,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAEvE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAY9B,eAAO,MAAM,iBAAiB,EAAE,MAyB/B,CAAC;AAEF,eAAe,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,MAAM,EAAE,MAAM,eAAe,CAAC;AAG3D,OAAO,uBAAuB,CAAC;AAa/B,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAEvE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAY9B,eAAO,MAAM,iBAAiB,EAAE,MAiC/B,CAAC;AAEF,eAAe,iBAAiB,CAAC"}

package/dist/index.js

@@ -2,9 +2,14 @@ import { logger } from "@elizaos/core";
 import { z } from "zod";
 import "./service-registry.js";
 import { computeruseClickAction } from "./actions/click.js";
+import { computeruseClickCoordinatesAction } from "./actions/click-coordinates.js";
+import { computeruseDragAction } from "./actions/drag.js";
 import { computeruseGetApplicationsAction } from "./actions/get-applications.js";
 import { computeruseGetWindowTreeAction } from "./actions/get-window-tree.js";
 import { computeruseOpenApplicationAction } from "./actions/open-application.js";
+import { computerusePressKeyAction } from "./actions/press-key.js";
+import { computeruseScreenshotAction } from "./actions/screenshot.js";
+import { computeruseScrollAction } from "./actions/scroll.js";
 import { computeruseTypeAction } from "./actions/type.js";
 import { computeruseAvailableAppsProvider } from "./providers/available-apps.js";
 import { computeruseStateProvider } from "./providers/computeruse-state.js";
@@ -24,13 +29,16 @@ export const computerusePlugin = {
     name: "plugin-computeruse",
     description: "Computer automation plugin (local or MCP)",
     config: {
-        COMPUTERUSE_ENABLED: process.env.COMPUTERUSE_ENABLED ?? "false",
+        COMPUTERUSE_ENABLED: process.env.COMPUTERUSE_ENABLED ?? "true",
         COMPUTERUSE_MODE: process.env.COMPUTERUSE_MODE ?? "auto",
         COMPUTERUSE_MCP_SERVER: process.env.COMPUTERUSE_MCP_SERVER ?? "computeruse",
     },
     async init(config, _runtime) {
         logger.info("Initializing computeruse plugin");
-        const validated = await configSchema.parseAsync(config);
+        // Auto-enable when the plugin is loaded — the plugin being present in
+        // the runtime means the user toggled it on via the Capabilities UI.
+        const configWithDefaults = { ...config, COMPUTERUSE_ENABLED: config.COMPUTERUSE_ENABLED ?? "true" };
+        const validated = await configSchema.parseAsync(configWithDefaults);
         for (const [key, value] of Object.entries(validated)) {
             process.env[key] = String(value);
         }
@@ -39,7 +47,12 @@ export const computerusePlugin = {
     actions: [
         computeruseOpenApplicationAction,
         computeruseClickAction,
+        computeruseClickCoordinatesAction,
         computeruseTypeAction,
+        computerusePressKeyAction,
+        computeruseScrollAction,
+        computeruseDragAction,
+        computeruseScreenshotAction,
         computeruseGetApplicationsAction,
         computeruseGetWindowTreeAction,
     ],

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,uBAAuB,CAAC;AAC/B,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,8BAA8B,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAEvE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,mBAAmB,EAAE,CAAC;SACnB,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,OAAO,CAAC,OAAO,CAAC;SAChB,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC;IACrC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7E,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC;CACrE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAW;IACvC,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,2CAA2C;IACxD,MAAM,EAAE;QACN,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO;QAC/D,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM;QACxD,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,aAAa;KAC5E;IACD,KAAK,CAAC,IAAI,CAAC,MAA8B,EAAE,QAAuB;QAChE,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,QAAQ,EAAE,CAAC,kBAAkB,CAAC;IAC9B,OAAO,EAAE;QACP,gCAAgC;QAChC,sBAAsB;QACtB,qBAAqB;QACrB,gCAAgC;QAChC,8BAA8B;KAC/B;IACD,SAAS,EAAE,CAAC,wBAAwB,EAAE,gCAAgC,CAAC;IACvE,YAAY,EAAE,CAAC,KAAK,CAAC;CACtB,CAAC;AAEF,eAAe,iBAAiB,CAAC","sourcesContent":["import type { IAgentRuntime, Plugin } from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport { z } from \"zod\";\nimport \"./service-registry.js\";\nimport { computeruseClickAction } from \"./actions/click.js\";\nimport { computeruseGetApplicationsAction } from \"./actions/get-applications.js\";\nimport { computeruseGetWindowTreeAction } from \"./actions/get-window-tree.js\";\nimport { computeruseOpenApplicationAction } from \"./actions/open-application.js\";\nimport { computeruseTypeAction } from \"./actions/type.js\";\nimport { computeruseAvailableAppsProvider } from \"./providers/available-apps.js\";\nimport { computeruseStateProvider } from \"./providers/computeruse-state.js\";\nimport { ComputerUseService } from \"./services/computeruse-service.js\";\n\nexport * from \"./types.js\";\nexport { ComputerUseService };\n\nconst configSchema = z.object({\n  COMPUTERUSE_ENABLED: z\n    .string()\n    .optional()\n    .default(\"false\")\n    .transform((val) => val === \"true\"),\n  COMPUTERUSE_MODE: z.enum([\"auto\", \"local\", \"mcp\"]).optional().default(\"auto\"),\n  COMPUTERUSE_MCP_SERVER: z.string().optional().default(\"computeruse\"),\n});\n\nexport const computerusePlugin: Plugin = {\n  name: \"plugin-computeruse\",\n  description: \"Computer automation plugin (local or MCP)\",\n  config: {\n    COMPUTERUSE_ENABLED: process.env.COMPUTERUSE_ENABLED ?? \"false\",\n    COMPUTERUSE_MODE: process.env.COMPUTERUSE_MODE ?? \"auto\",\n    COMPUTERUSE_MCP_SERVER: process.env.COMPUTERUSE_MCP_SERVER ?? \"computeruse\",\n  },\n  async init(config: Record<string, string>, _runtime: IAgentRuntime): Promise<void> {\n    logger.info(\"Initializing computeruse plugin\");\n    const validated = await configSchema.parseAsync(config);\n    for (const [key, value] of Object.entries(validated)) {\n      process.env[key] = String(value);\n    }\n  },\n  services: [ComputerUseService],\n  actions: [\n    computeruseOpenApplicationAction,\n    computeruseClickAction,\n    computeruseTypeAction,\n    computeruseGetApplicationsAction,\n    computeruseGetWindowTreeAction,\n  ],\n  providers: [computeruseStateProvider, computeruseAvailableAppsProvider],\n  dependencies: [\"mcp\"],\n};\n\nexport default computerusePlugin;\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,uBAAuB,CAAC;AAC/B,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,iCAAiC,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,8BAA8B,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAEvE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,mBAAmB,EAAE,CAAC;SACnB,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,OAAO,CAAC,OAAO,CAAC;SAChB,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC;IACrC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7E,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC;CACrE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAW;IACvC,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,2CAA2C;IACxD,MAAM,EAAE;QACN,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,MAAM;QAC9D,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM;QACxD,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,aAAa;KAC5E;IACD,KAAK,CAAC,IAAI,CAAC,MAA8B,EAAE,QAAuB;QAChE,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAC/C,sEAAsE;QACtE,oEAAoE;QACpE,MAAM,kBAAkB,GAAG,EAAE,GAAG,MAAM,EAAE,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,IAAI,MAAM,EAAE,CAAC;QACpG,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACpE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,QAAQ,EAAE,CAAC,kBAAkB,CAAC;IAC9B,OAAO,EAAE;QACP,gCAAgC;QAChC,sBAAsB;QACtB,iCAAiC;QACjC,qBAAqB;QACrB,yBAAyB;QACzB,uBAAuB;QACvB,qBAAqB;QACrB,2BAA2B;QAC3B,gCAAgC;QAChC,8BAA8B;KAC/B;IACD,SAAS,EAAE,CAAC,wBAAwB,EAAE,gCAAgC,CAAC;IACvE,YAAY,EAAE,CAAC,KAAK,CAAC;CACtB,CAAC;AAEF,eAAe,iBAAiB,CAAC","sourcesContent":["import type { IAgentRuntime, Plugin } from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport { z } from \"zod\";\nimport \"./service-registry.js\";\nimport { computeruseClickAction } from \"./actions/click.js\";\nimport { computeruseClickCoordinatesAction } from \"./actions/click-coordinates.js\";\nimport { computeruseDragAction } from \"./actions/drag.js\";\nimport { computeruseGetApplicationsAction } from \"./actions/get-applications.js\";\nimport { computeruseGetWindowTreeAction } from \"./actions/get-window-tree.js\";\nimport { computeruseOpenApplicationAction } from \"./actions/open-application.js\";\nimport { computerusePressKeyAction } from \"./actions/press-key.js\";\nimport { computeruseScreenshotAction } from \"./actions/screenshot.js\";\nimport { computeruseScrollAction } from \"./actions/scroll.js\";\nimport { computeruseTypeAction } from \"./actions/type.js\";\nimport { computeruseAvailableAppsProvider } from \"./providers/available-apps.js\";\nimport { computeruseStateProvider } from \"./providers/computeruse-state.js\";\nimport { ComputerUseService } from \"./services/computeruse-service.js\";\n\nexport * from \"./types.js\";\nexport { ComputerUseService };\n\nconst configSchema = z.object({\n  COMPUTERUSE_ENABLED: z\n    .string()\n    .optional()\n    .default(\"false\")\n    .transform((val) => val === \"true\"),\n  COMPUTERUSE_MODE: z.enum([\"auto\", \"local\", \"mcp\"]).optional().default(\"auto\"),\n  COMPUTERUSE_MCP_SERVER: z.string().optional().default(\"computeruse\"),\n});\n\nexport const computerusePlugin: Plugin = {\n  name: \"plugin-computeruse\",\n  description: \"Computer automation plugin (local or MCP)\",\n  config: {\n    COMPUTERUSE_ENABLED: process.env.COMPUTERUSE_ENABLED ?? \"true\",\n    COMPUTERUSE_MODE: process.env.COMPUTERUSE_MODE ?? \"auto\",\n    COMPUTERUSE_MCP_SERVER: process.env.COMPUTERUSE_MCP_SERVER ?? \"computeruse\",\n  },\n  async init(config: Record<string, string>, _runtime: IAgentRuntime): Promise<void> {\n    logger.info(\"Initializing computeruse plugin\");\n    // Auto-enable when the plugin is loaded — the plugin being present in\n    // the runtime means the user toggled it on via the Capabilities UI.\n    const configWithDefaults = { ...config, COMPUTERUSE_ENABLED: config.COMPUTERUSE_ENABLED ?? \"true\" };\n    const validated = await configSchema.parseAsync(configWithDefaults);\n    for (const [key, value] of Object.entries(validated)) {\n      process.env[key] = String(value);\n    }\n  },\n  services: [ComputerUseService],\n  actions: [\n    computeruseOpenApplicationAction,\n    computeruseClickAction,\n    computeruseClickCoordinatesAction,\n    computeruseTypeAction,\n    computerusePressKeyAction,\n    computeruseScrollAction,\n    computeruseDragAction,\n    computeruseScreenshotAction,\n    computeruseGetApplicationsAction,\n    computeruseGetWindowTreeAction,\n  ],\n  providers: [computeruseStateProvider, computeruseAvailableAppsProvider],\n  dependencies: [\"mcp\"],\n};\n\nexport default computerusePlugin;\n"]}

package/dist/services/computeruse-service.d.ts

@@ -26,6 +26,11 @@ export declare class ComputerUseService extends Service {
     click(selector: string, timeoutMs: number, process?: string): Promise<void>;
     typeText(selector: string, text: string, timeoutMs: number, clearBeforeTyping: boolean, process?: string): Promise<void>;
     getWindowTree(process: string, title?: string, maxDepth?: number): Promise<string>;
+    clickCoordinates(x: number, y: number, button?: string, numClicks?: number): Promise<void>;
+    pressKey(keys: string): Promise<void>;
+    scroll(direction?: string, amount?: number, x?: number, y?: number): Promise<void>;
+    drag(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+    screenshot(): Promise<string>;
     getApplications(): Promise<string[]>;
 }
 //# sourceMappingURL=computeruse-service.d.ts.map

package/dist/services/computeruse-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"computeruse-service.d.ts","sourceRoot":"","sources":["../../src/services/computeruse-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAU,OAAO,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EACL,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EAGvB,MAAM,aAAa,CAAC;AAYrB,qBAAa,kBAAmB,SAAQ,OAAO;IAC7C,MAAM,CAAC,WAAW,SAAiB;IACnC,qBAAqB,SACmF;IAExG,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,WAAW,CAAS;IAG5B,OAAO,CAAC,YAAY,CAAuD;gBAE/D,OAAO,CAAC,EAAE,aAAa;WAStB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAKjE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,OAAO,IAAI,iBAAiB,CAAC,kBAAkB,CAAC;IAIhD,cAAc,IAAI,sBAAsB,GAAG,IAAI;IAI/C,gBAAgB,IAAI,MAAM;IAI1B,SAAS,IAAI,OAAO;IAId,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;YAMpB,iBAAiB;YAiCjB,kBAAkB;YAuBlB,gBAAgB;YAahB,iBAAiB;IAa/B,OAAO,CAAC,MAAM;IAQd,OAAO,CAAC,0BAA0B;IA0B5B,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB/C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B3E,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,OAAO,EAC1B,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;IA8BV,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAiClF,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAiC3C"}
+{"version":3,"file":"computeruse-service.d.ts","sourceRoot":"","sources":["../../src/services/computeruse-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAU,OAAO,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EACL,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EAGvB,MAAM,aAAa,CAAC;AAYrB,qBAAa,kBAAmB,SAAQ,OAAO;IAC7C,MAAM,CAAC,WAAW,SAAiB;IACnC,qBAAqB,SACmF;IAExG,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,WAAW,CAAS;IAG5B,OAAO,CAAC,YAAY,CAAuD;gBAE/D,OAAO,CAAC,EAAE,aAAa;WAStB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAKjE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,OAAO,IAAI,iBAAiB,CAAC,kBAAkB,CAAC;IAIhD,cAAc,IAAI,sBAAsB,GAAG,IAAI;IAI/C,gBAAgB,IAAI,MAAM;IAI1B,SAAS,IAAI,OAAO;IAId,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;YAMpB,iBAAiB;YAgDjB,kBAAkB;YAuBlB,gBAAgB;YAahB,iBAAiB;IAa/B,OAAO,CAAC,MAAM;IAQd,OAAO,CAAC,0BAA0B;IA0B5B,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB/C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B3E,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,OAAO,EAC1B,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;IA8BV,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAiClF,gBAAgB,CACpB,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,MAAM,GAAE,MAAe,EACvB,SAAS,GAAE,MAAU,GACpB,OAAO,CAAC,IAAI,CAAC;IAuBV,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBrC,MAAM,CACV,SAAS,GAAE,MAAe,EAC1B,MAAM,GAAE,MAAU,EAClB,CAAC,CAAC,EAAE,MAAM,EACV,CAAC,CAAC,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC;IAiCV,IAAI,CACR,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IA0BV,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IA8B7B,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAiC3C"}

package/dist/services/computeruse-service.js

@@ -49,6 +49,16 @@ export class ComputerUseService extends Service {
             this.backendName = null;
             return;
         }
+        // ── Sandbox mode: force MCP to prevent local desktop control ──────
+        const isSandboxMode = Boolean(this.runtime && this.runtime.sandboxMode);
+        if (isSandboxMode) {
+            logger.info("[computeruse] sandbox mode active — forcing MCP backend (local desktop control disabled)");
+            this.computeruseConfig.COMPUTERUSE_MODE = "mcp";
+            await this.ensureMcpBackend();
+            this.backendName = "mcp";
+            return;
+        }
+        // ── End sandbox mode override ─────────────────────────────────────
         const mode = this.computeruseConfig.COMPUTERUSE_MODE;
         if (mode === "local") {
             await this.ensureLocalBackend();
@@ -242,6 +252,108 @@ export class ComputerUseService extends Service {
         }
         throw new Error("ComputerUse backend not initialized");
     }
+    async clickCoordinates(x, y, button = "left", numClicks = 1) {
+        await this.ensureReady();
+        if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
+            throw new Error("ComputerUse is disabled");
+        if (this.backendName === "local") {
+            await this.ensureLocalBackend();
+            this.localDesktop?.clickAtCoordinates(x, y);
+            return;
+        }
+        if (this.backendName === "mcp") {
+            const mcp = this.getMcp();
+            await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "click_at_coordinates", { x, y, button, num_clicks: numClicks });
+            return;
+        }
+        throw new Error("ComputerUse backend not initialized");
+    }
+    async pressKey(keys) {
+        await this.ensureReady();
+        if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
+            throw new Error("ComputerUse is disabled");
+        if (this.backendName === "local") {
+            await this.ensureLocalBackend();
+            this.localDesktop?.pressKey(keys);
+            return;
+        }
+        if (this.backendName === "mcp") {
+            const mcp = this.getMcp();
+            await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "press_key", { key: keys });
+            return;
+        }
+        throw new Error("ComputerUse backend not initialized");
+    }
+    async scroll(direction = "down", amount = 3, x, y) {
+        await this.ensureReady();
+        if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
+            throw new Error("ComputerUse is disabled");
+        if (this.backendName === "local") {
+            await this.ensureLocalBackend();
+            // Local backend: click at position first if specified, then scroll via key presses
+            if (x !== undefined && y !== undefined) {
+                this.localDesktop?.clickAtCoordinates(x, y);
+            }
+            const scrollKey = direction === "up" ? "scrollup" : "scrolldown";
+            for (let i = 0; i < amount; i++) {
+                this.localDesktop?.pressKey(scrollKey);
+            }
+            return;
+        }
+        if (this.backendName === "mcp") {
+            const mcp = this.getMcp();
+            const toolArgs = { direction, amount };
+            if (x !== undefined)
+                toolArgs.x = x;
+            if (y !== undefined)
+                toolArgs.y = y;
+            await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "scroll", toolArgs);
+            return;
+        }
+        throw new Error("ComputerUse backend not initialized");
+    }
+    async drag(startX, startY, endX, endY) {
+        await this.ensureReady();
+        if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
+            throw new Error("ComputerUse is disabled");
+        if (this.backendName === "local") {
+            await this.ensureLocalBackend();
+            // The Desktop native API does not expose a drag method.
+            // Local backend can only click the start position; full drag requires MCP.
+            logger.warn("[computeruse] Local drag is limited to click-at-start. Use MCP for full drag.");
+            this.localDesktop?.clickAtCoordinates(startX, startY);
+            return;
+        }
+        if (this.backendName === "mcp") {
+            const mcp = this.getMcp();
+            await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "drag", { start_x: startX, start_y: startY, end_x: endX, end_y: endY });
+            return;
+        }
+        throw new Error("ComputerUse backend not initialized");
+    }
+    async screenshot() {
+        await this.ensureReady();
+        if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
+            throw new Error("ComputerUse is disabled");
+        if (this.backendName === "local") {
+            await this.ensureLocalBackend();
+            const result = this.localDesktop?.captureMonitor(0);
+            if (result && typeof result === "object" && "toBase64" in result) {
+                return result.toBase64();
+            }
+            return String(result ?? "");
+        }
+        if (this.backendName === "mcp") {
+            const mcp = this.getMcp();
+            const res = await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "capture_screenshot", {});
+            const texts = res.content
+                .filter((c) => c.type === "text")
+                .map((c) => c.text)
+                .filter((t) => typeof t === "string");
+            return texts.join("");
+        }
+        throw new Error("ComputerUse backend not initialized");
+    }
     async getApplications() {
         await this.ensureReady();
         if (!this.computeruseConfig.COMPUTERUSE_ENABLED)

package/dist/services/computeruse-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"computeruse-service.js","sourceRoot":"","sources":["../../src/services/computeruse-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAGL,uBAAuB,GAExB,MAAM,aAAa,CAAC;AAYrB,MAAM,OAAO,kBAAmB,SAAQ,OAAO;IAC7C,MAAM,CAAC,WAAW,GAAG,aAAa,CAAC;IACnC,qBAAqB,GACnB,qGAAqG,CAAC;IAEhG,iBAAiB,CAAoB;IACrC,WAAW,GAAkC,IAAI,CAAC;IAClD,WAAW,GAAG,KAAK,CAAC;IAE5B,uEAAuE;IAC/D,YAAY,GAAkD,IAAI,CAAC;IAE3E,YAAY,OAAuB;QACjC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,KAAK,CAAC;YACrD,mBAAmB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC;YACvE,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC;YAChE,sBAAsB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,aAAa,CAAC;SACpF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAsB;QACvC,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;IACjD,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC;IACvD,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;QACrD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,OAAO;QACT,CAAC;QAED,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,iEAAiE,GAAG,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAE9B,kDAAkD;QAClD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YACjD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QACxC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YAClC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,sEAAsE,CAClH,CAAC;YACJ,CAAC;iBAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,qEAAqE,CACjH,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,eAAe,UAAU,0EAA0E,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,oFAAoF;QACpF,kEAAkE;QAClE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAiB,KAAK,CAAC,CAAC;YAC3D,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;YACpB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;IACJ,CAAC;IAEO,MAAM;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAiB,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,0BAA0B,CAChC,WAAmB,EACnB,WAAoB;QAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxE,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,kBAAkB,EAAE;gBACpF,QAAQ,EAAE,OAAO;gBACjB,qBAAqB,EAAE,EAAE;gBACzB,yBAAyB,EAAE,EAAE;gBAC7B,yBAAyB,EAAE,KAAK;aACjC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,SAAiB,EAAE,OAAgB;QAC/D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC3D,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClE,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,eAAe,EAAE;gBACjF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE,SAAS;gBACrB,qBAAqB,EAAE,EAAE;gBACzB,yBAAyB,EAAE,EAAE;gBAC7B,uBAAuB,EAAE,KAAK;gBAC9B,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,QAAgB,EAChB,IAAY,EACZ,SAAiB,EACjB,iBAA0B,EAC1B,OAAgB;QAEhB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC3D,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACzC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClE,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,mBAAmB,EAAE;gBACrF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,YAAY,EAAE,IAAI;gBAClB,UAAU,EAAE,SAAS;gBACrB,mBAAmB,EAAE,iBAAiB;gBACtC,uBAAuB,EAAE,KAAK;gBAC9B,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,KAAc,EAAE,QAAiB;QACpE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAe;gBAC3B,OAAO;gBACP,yBAAyB,EAAE,IAAI;aAChC,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;YAChD,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC;YAC/D,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,iBAAiB,EACjB,QAAQ,CACT,CAAC;YACF,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACrD,uCAAuC;YACvC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;gBACrB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,mCAAmC,EACnC,EAAE,CACH,CAAC;YACF,+FAA+F;YAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC","sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { logger, Service } from \"@elizaos/core\";\nimport type { CallToolResult } from \"@modelcontextprotocol/sdk/types.js\";\nimport {\n  type ComputerUseBackendName,\n  type ComputerUseConfig,\n  computerUseConfigSchema,\n  type JsonObject,\n} from \"../types.js\";\n\n// Runtime service shape (from @elizaos/plugin-mcp), referenced structurally to avoid a hard dep.\ninterface McpServiceLike extends Service {\n  callTool(\n    serverName: string,\n    toolName: string,\n    toolArguments?: Readonly<JsonObject>\n  ): Promise<CallToolResult>;\n  getServers(): ReadonlyArray<{ name: string; status: string }>;\n}\n\nexport class ComputerUseService extends Service {\n  static serviceType = \"computeruse\";\n  capabilityDescription =\n    \"Enables the agent to control a computer UI locally (when supported) or via a ComputerUse MCP server\";\n\n  private computeruseConfig: ComputerUseConfig;\n  private backendName: ComputerUseBackendName | null = null;\n  private initialized = false;\n\n  // Lazy-loaded to avoid importing native bindings unless actually used.\n  private localDesktop: import(\"@elizaos/computeruse\").Desktop | null = null;\n\n  constructor(runtime?: IAgentRuntime) {\n    super(runtime);\n    this.computeruseConfig = computerUseConfigSchema.parse({\n      COMPUTERUSE_ENABLED: String(process.env.COMPUTERUSE_ENABLED ?? \"false\"),\n      COMPUTERUSE_MODE: String(process.env.COMPUTERUSE_MODE ?? \"auto\"),\n      COMPUTERUSE_MCP_SERVER: String(process.env.COMPUTERUSE_MCP_SERVER ?? \"computeruse\"),\n    });\n  }\n\n  static async start(runtime: IAgentRuntime): Promise<ComputerUseService> {\n    const service = new ComputerUseService(runtime);\n    return service;\n  }\n\n  async stop(): Promise<void> {\n    this.localDesktop = null;\n    this.backendName = null;\n    this.initialized = false;\n  }\n\n  getMode(): ComputerUseConfig[\"COMPUTERUSE_MODE\"] {\n    return this.computeruseConfig.COMPUTERUSE_MODE;\n  }\n\n  getBackendName(): ComputerUseBackendName | null {\n    return this.backendName;\n  }\n\n  getMcpServerName(): string {\n    return this.computeruseConfig.COMPUTERUSE_MCP_SERVER;\n  }\n\n  isEnabled(): boolean {\n    return this.computeruseConfig.COMPUTERUSE_ENABLED;\n  }\n\n  async ensureReady(): Promise<void> {\n    if (this.initialized) return;\n    this.initialized = true;\n    await this.initializeBackend();\n  }\n\n  private async initializeBackend(): Promise<void> {\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) {\n      logger.info(\"[computeruse] disabled (COMPUTERUSE_ENABLED=false)\");\n      this.backendName = null;\n      return;\n    }\n\n    const mode = this.computeruseConfig.COMPUTERUSE_MODE;\n    if (mode === \"local\") {\n      await this.ensureLocalBackend();\n      this.backendName = \"local\";\n      return;\n    }\n    if (mode === \"mcp\") {\n      await this.ensureMcpBackend();\n      this.backendName = \"mcp\";\n      return;\n    }\n\n    // auto - try local on all platforms, fall back to MCP\n    try {\n      await this.ensureLocalBackend();\n      this.backendName = \"local\";\n      return;\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.warn(`[computeruse] local backend unavailable, falling back to mcp: ${msg}`);\n    }\n\n    await this.ensureMcpBackend();\n    this.backendName = \"mcp\";\n  }\n\n  private async ensureLocalBackend(): Promise<void> {\n    if (this.localDesktop) return;\n\n    // Import only when needed (native optional deps).\n    try {\n      const mod = await import(\"@elizaos/computeruse\");\n      this.localDesktop = new mod.Desktop();\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      const platform = process.platform;\n      if (platform === \"darwin\") {\n        throw new Error(\n          `macOS native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-darwin-* is installed.`\n        );\n      } else if (platform === \"linux\") {\n        throw new Error(\n          `Linux native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-linux-* is installed.`\n        );\n      }\n      throw err;\n    }\n  }\n\n  private async ensureMcpBackend(): Promise<void> {\n    const mcp = await this.waitForMcpService();\n\n    const serverName = this.computeruseConfig.COMPUTERUSE_MCP_SERVER;\n    const servers = mcp.getServers();\n    const exists = servers.some((s) => s.name === serverName);\n    if (!exists) {\n      throw new Error(\n        `MCP server \"${serverName}\" not configured. Add it under runtime/character settings \"mcp.servers\".`\n      );\n    }\n  }\n\n  private async waitForMcpService(): Promise<McpServiceLike> {\n    // Services are registered during runtime initialization; depending on plugin order,\n    // \"mcp\" may not be available during another service's start hook.\n    for (let attempt = 0; attempt < 20; attempt++) {\n      const mcp = this.runtime.getService<McpServiceLike>(\"mcp\");\n      if (mcp) return mcp;\n      await new Promise<void>((resolve) => setTimeout(resolve, 50));\n    }\n    throw new Error(\n      \"MCP service not available. Add @elizaos/plugin-mcp and configure a computeruse server.\"\n    );\n  }\n\n  private getMcp(): McpServiceLike {\n    const mcp = this.runtime.getService<McpServiceLike>(\"mcp\");\n    if (!mcp) {\n      throw new Error(\"MCP service not available\");\n    }\n    return mcp;\n  }\n\n  private parseProcessScopedSelector(\n    rawSelector: string,\n    processHint?: string\n  ): { process: string; selector: string } {\n    const selector = rawSelector.trim();\n    const match = selector.match(/^\\s*process:([^\\s>]+)\\s*(?:>>\\s*(.*))?$/);\n    if (match) {\n      const process = match[1]?.trim();\n      const inner = (match[2] ?? \"\").trim();\n      if (!process) {\n        throw new Error(\n          \"Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'\"\n        );\n      }\n      return { process, selector: inner };\n    }\n\n    const process = processHint?.trim();\n    if (!process) {\n      throw new Error(\n        \"Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'\"\n      );\n    }\n    return { process, selector };\n  }\n\n  async openApplication(appName: string): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      this.localDesktop?.openApplication(appName);\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"open_application\", {\n        app_name: appName,\n        verify_element_exists: \"\",\n        verify_element_not_exists: \"\",\n        include_tree_after_action: false,\n      });\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async click(selector: string, timeoutMs: number, process?: string): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const el = await this.localDesktop?.locator(selector).first(timeoutMs);\n      if (!el) throw new Error(`Element not found: ${selector}`);\n      await el.click();\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const parsed = this.parseProcessScopedSelector(selector, process);\n      await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"click_element\", {\n        process: parsed.process,\n        selector: parsed.selector,\n        timeout_ms: timeoutMs,\n        verify_element_exists: \"\",\n        verify_element_not_exists: \"\",\n        highlight_before_action: false,\n        ui_diff_before_after: false,\n      });\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async typeText(\n    selector: string,\n    text: string,\n    timeoutMs: number,\n    clearBeforeTyping: boolean,\n    process?: string\n  ): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const el = await this.localDesktop?.locator(selector).first(timeoutMs);\n      if (!el) throw new Error(`Element not found: ${selector}`);\n      el.typeText(text, { clearBeforeTyping });\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const parsed = this.parseProcessScopedSelector(selector, process);\n      await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"type_into_element\", {\n        process: parsed.process,\n        selector: parsed.selector,\n        text_to_type: text,\n        timeout_ms: timeoutMs,\n        clear_before_typing: clearBeforeTyping,\n        highlight_before_action: false,\n        ui_diff_before_after: false,\n      });\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async getWindowTree(process: string, title?: string, maxDepth?: number): Promise<string> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const tree = this.localDesktop?.getWindowTree(process, title, undefined);\n      return JSON.stringify(tree ?? null, null, 2);\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const toolArgs: JsonObject = {\n        process,\n        include_tree_after_action: true,\n      };\n      if (title !== undefined) toolArgs.title = title;\n      if (maxDepth !== undefined) toolArgs.tree_max_depth = maxDepth;\n      const res = await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"get_window_tree\",\n        toolArgs\n      );\n      const texts = res.content\n        .filter((c) => c.type === \"text\")\n        .map((c) => c.text)\n        .filter((t) => typeof t === \"string\");\n      return texts.join(\"\\n\");\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async getApplications(): Promise<string[]> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const apps = this.localDesktop?.applications() ?? [];\n      // Normalize into human-readable names.\n      const names: string[] = [];\n      for (const app of apps) {\n        const n = app.name();\n        if (typeof n === \"string\" && n.trim().length > 0) names.push(n);\n      }\n      return names;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const res = await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"get_applications_and_windows_list\",\n        {}\n      );\n      // Return text-only summary for now; structured parsing is done in actions/providers if needed.\n      const texts = res.content\n        .filter((c) => c.type === \"text\")\n        .map((c) => c.text)\n        .filter((t) => typeof t === \"string\");\n      return texts.length > 0 ? texts : [\"(see MCP tool output)\"];\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n}\n"]}
+{"version":3,"file":"computeruse-service.js","sourceRoot":"","sources":["../../src/services/computeruse-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAGL,uBAAuB,GAExB,MAAM,aAAa,CAAC;AAYrB,MAAM,OAAO,kBAAmB,SAAQ,OAAO;IAC7C,MAAM,CAAC,WAAW,GAAG,aAAa,CAAC;IACnC,qBAAqB,GACnB,qGAAqG,CAAC;IAEhG,iBAAiB,CAAoB;IACrC,WAAW,GAAkC,IAAI,CAAC;IAClD,WAAW,GAAG,KAAK,CAAC;IAE5B,uEAAuE;IAC/D,YAAY,GAAkD,IAAI,CAAC;IAE3E,YAAY,OAAuB;QACjC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,KAAK,CAAC;YACrD,mBAAmB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC;YACvE,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC;YAChE,sBAAsB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,aAAa,CAAC;SACpF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAsB;QACvC,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;IACjD,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC;IACvD,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,aAAa,GAAG,OAAO,CAC3B,IAAI,CAAC,OAAO,IAAK,IAAI,CAAC,OAAmC,CAAC,WAAW,CACtE,CAAC;QACF,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CACT,0FAA0F,CAC3F,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChD,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,OAAO;QACT,CAAC;QACD,qEAAqE;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;QACrD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,OAAO;QACT,CAAC;QAED,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,iEAAiE,GAAG,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAE9B,kDAAkD;QAClD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YACjD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QACxC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YAClC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,sEAAsE,CAClH,CAAC;YACJ,CAAC;iBAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,qEAAqE,CACjH,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,eAAe,UAAU,0EAA0E,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,oFAAoF;QACpF,kEAAkE;QAClE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAiB,KAAK,CAAC,CAAC;YAC3D,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;YACpB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;IACJ,CAAC;IAEO,MAAM;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAiB,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,0BAA0B,CAChC,WAAmB,EACnB,WAAoB;QAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxE,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,kBAAkB,EAAE;gBACpF,QAAQ,EAAE,OAAO;gBACjB,qBAAqB,EAAE,EAAE;gBACzB,yBAAyB,EAAE,EAAE;gBAC7B,yBAAyB,EAAE,KAAK;aACjC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,SAAiB,EAAE,OAAgB;QAC/D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC3D,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClE,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,eAAe,EAAE;gBACjF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE,SAAS;gBACrB,qBAAqB,EAAE,EAAE;gBACzB,yBAAyB,EAAE,EAAE;gBAC7B,uBAAuB,EAAE,KAAK;gBAC9B,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,QAAgB,EAChB,IAAY,EACZ,SAAiB,EACjB,iBAA0B,EAC1B,OAAgB;QAEhB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC3D,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACzC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClE,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,mBAAmB,EAAE;gBACrF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,YAAY,EAAE,IAAI;gBAClB,UAAU,EAAE,SAAS;gBACrB,mBAAmB,EAAE,iBAAiB;gBACtC,uBAAuB,EAAE,KAAK;gBAC9B,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,KAAc,EAAE,QAAiB;QACpE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAe;gBAC3B,OAAO;gBACP,yBAAyB,EAAE,IAAI;aAChC,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;YAChD,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC;YAC/D,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,iBAAiB,EACjB,QAAQ,CACT,CAAC;YACF,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,CAAS,EACT,CAAS,EACT,SAAiB,MAAM,EACvB,YAAoB,CAAC;QAErB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,sBAAsB,EACtB,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CACxC,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAClC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,WAAW,EACX,EAAE,GAAG,EAAE,IAAI,EAAE,CACd,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,MAAM,CACV,YAAoB,MAAM,EAC1B,SAAiB,CAAC,EAClB,CAAU,EACV,CAAU;QAEV,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,mFAAmF;YACnF,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gBACvC,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9C,CAAC;YACD,MAAM,SAAS,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;YACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzC,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAe,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACnD,IAAI,CAAC,KAAK,SAAS;gBAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,SAAS;gBAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,QAAQ,EACR,QAAQ,CACT,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAc,EACd,MAAc,EACd,IAAY,EACZ,IAAY;QAEZ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,wDAAwD;YACxD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;YAC7F,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,MAAM,EACN,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAC/D,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;YACpD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,UAAU,IAAI,MAAM,EAAE,CAAC;gBACjE,OAAQ,MAAiC,CAAC,QAAQ,EAAE,CAAC;YACvD,CAAC;YACD,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,oBAAoB,EACpB,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACrD,uCAAuC;YACvC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;gBACrB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,mCAAmC,EACnC,EAAE,CACH,CAAC;YACF,+FAA+F;YAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC","sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { logger, Service } from \"@elizaos/core\";\nimport type { CallToolResult } from \"@modelcontextprotocol/sdk/types.js\";\nimport {\n  type ComputerUseBackendName,\n  type ComputerUseConfig,\n  computerUseConfigSchema,\n  type JsonObject,\n} from \"../types.js\";\n\n// Runtime service shape (from @elizaos/plugin-mcp), referenced structurally to avoid a hard dep.\ninterface McpServiceLike extends Service {\n  callTool(\n    serverName: string,\n    toolName: string,\n    toolArguments?: Readonly<JsonObject>\n  ): Promise<CallToolResult>;\n  getServers(): ReadonlyArray<{ name: string; status: string }>;\n}\n\nexport class ComputerUseService extends Service {\n  static serviceType = \"computeruse\";\n  capabilityDescription =\n    \"Enables the agent to control a computer UI locally (when supported) or via a ComputerUse MCP server\";\n\n  private computeruseConfig: ComputerUseConfig;\n  private backendName: ComputerUseBackendName | null = null;\n  private initialized = false;\n\n  // Lazy-loaded to avoid importing native bindings unless actually used.\n  private localDesktop: import(\"@elizaos/computeruse\").Desktop | null = null;\n\n  constructor(runtime?: IAgentRuntime) {\n    super(runtime);\n    this.computeruseConfig = computerUseConfigSchema.parse({\n      COMPUTERUSE_ENABLED: String(process.env.COMPUTERUSE_ENABLED ?? \"false\"),\n      COMPUTERUSE_MODE: String(process.env.COMPUTERUSE_MODE ?? \"auto\"),\n      COMPUTERUSE_MCP_SERVER: String(process.env.COMPUTERUSE_MCP_SERVER ?? \"computeruse\"),\n    });\n  }\n\n  static async start(runtime: IAgentRuntime): Promise<ComputerUseService> {\n    const service = new ComputerUseService(runtime);\n    return service;\n  }\n\n  async stop(): Promise<void> {\n    this.localDesktop = null;\n    this.backendName = null;\n    this.initialized = false;\n  }\n\n  getMode(): ComputerUseConfig[\"COMPUTERUSE_MODE\"] {\n    return this.computeruseConfig.COMPUTERUSE_MODE;\n  }\n\n  getBackendName(): ComputerUseBackendName | null {\n    return this.backendName;\n  }\n\n  getMcpServerName(): string {\n    return this.computeruseConfig.COMPUTERUSE_MCP_SERVER;\n  }\n\n  isEnabled(): boolean {\n    return this.computeruseConfig.COMPUTERUSE_ENABLED;\n  }\n\n  async ensureReady(): Promise<void> {\n    if (this.initialized) return;\n    this.initialized = true;\n    await this.initializeBackend();\n  }\n\n  private async initializeBackend(): Promise<void> {\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) {\n      logger.info(\"[computeruse] disabled (COMPUTERUSE_ENABLED=false)\");\n      this.backendName = null;\n      return;\n    }\n\n    // ── Sandbox mode: force MCP to prevent local desktop control ──────\n    const isSandboxMode = Boolean(\n      this.runtime && (this.runtime as Record<string, unknown>).sandboxMode,\n    );\n    if (isSandboxMode) {\n      logger.info(\n        \"[computeruse] sandbox mode active — forcing MCP backend (local desktop control disabled)\",\n      );\n      this.computeruseConfig.COMPUTERUSE_MODE = \"mcp\";\n      await this.ensureMcpBackend();\n      this.backendName = \"mcp\";\n      return;\n    }\n    // ── End sandbox mode override ─────────────────────────────────────\n\n    const mode = this.computeruseConfig.COMPUTERUSE_MODE;\n    if (mode === \"local\") {\n      await this.ensureLocalBackend();\n      this.backendName = \"local\";\n      return;\n    }\n    if (mode === \"mcp\") {\n      await this.ensureMcpBackend();\n      this.backendName = \"mcp\";\n      return;\n    }\n\n    // auto - try local on all platforms, fall back to MCP\n    try {\n      await this.ensureLocalBackend();\n      this.backendName = \"local\";\n      return;\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.warn(`[computeruse] local backend unavailable, falling back to mcp: ${msg}`);\n    }\n\n    await this.ensureMcpBackend();\n    this.backendName = \"mcp\";\n  }\n\n  private async ensureLocalBackend(): Promise<void> {\n    if (this.localDesktop) return;\n\n    // Import only when needed (native optional deps).\n    try {\n      const mod = await import(\"@elizaos/computeruse\");\n      this.localDesktop = new mod.Desktop();\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      const platform = process.platform;\n      if (platform === \"darwin\") {\n        throw new Error(\n          `macOS native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-darwin-* is installed.`\n        );\n      } else if (platform === \"linux\") {\n        throw new Error(\n          `Linux native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-linux-* is installed.`\n        );\n      }\n      throw err;\n    }\n  }\n\n  private async ensureMcpBackend(): Promise<void> {\n    const mcp = await this.waitForMcpService();\n\n    const serverName = this.computeruseConfig.COMPUTERUSE_MCP_SERVER;\n    const servers = mcp.getServers();\n    const exists = servers.some((s) => s.name === serverName);\n    if (!exists) {\n      throw new Error(\n        `MCP server \"${serverName}\" not configured. Add it under runtime/character settings \"mcp.servers\".`\n      );\n    }\n  }\n\n  private async waitForMcpService(): Promise<McpServiceLike> {\n    // Services are registered during runtime initialization; depending on plugin order,\n    // \"mcp\" may not be available during another service's start hook.\n    for (let attempt = 0; attempt < 20; attempt++) {\n      const mcp = this.runtime.getService<McpServiceLike>(\"mcp\");\n      if (mcp) return mcp;\n      await new Promise<void>((resolve) => setTimeout(resolve, 50));\n    }\n    throw new Error(\n      \"MCP service not available. Add @elizaos/plugin-mcp and configure a computeruse server.\"\n    );\n  }\n\n  private getMcp(): McpServiceLike {\n    const mcp = this.runtime.getService<McpServiceLike>(\"mcp\");\n    if (!mcp) {\n      throw new Error(\"MCP service not available\");\n    }\n    return mcp;\n  }\n\n  private parseProcessScopedSelector(\n    rawSelector: string,\n    processHint?: string\n  ): { process: string; selector: string } {\n    const selector = rawSelector.trim();\n    const match = selector.match(/^\\s*process:([^\\s>]+)\\s*(?:>>\\s*(.*))?$/);\n    if (match) {\n      const process = match[1]?.trim();\n      const inner = (match[2] ?? \"\").trim();\n      if (!process) {\n        throw new Error(\n          \"Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'\"\n        );\n      }\n      return { process, selector: inner };\n    }\n\n    const process = processHint?.trim();\n    if (!process) {\n      throw new Error(\n        \"Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'\"\n      );\n    }\n    return { process, selector };\n  }\n\n  async openApplication(appName: string): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      this.localDesktop?.openApplication(appName);\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"open_application\", {\n        app_name: appName,\n        verify_element_exists: \"\",\n        verify_element_not_exists: \"\",\n        include_tree_after_action: false,\n      });\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async click(selector: string, timeoutMs: number, process?: string): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const el = await this.localDesktop?.locator(selector).first(timeoutMs);\n      if (!el) throw new Error(`Element not found: ${selector}`);\n      await el.click();\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const parsed = this.parseProcessScopedSelector(selector, process);\n      await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"click_element\", {\n        process: parsed.process,\n        selector: parsed.selector,\n        timeout_ms: timeoutMs,\n        verify_element_exists: \"\",\n        verify_element_not_exists: \"\",\n        highlight_before_action: false,\n        ui_diff_before_after: false,\n      });\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async typeText(\n    selector: string,\n    text: string,\n    timeoutMs: number,\n    clearBeforeTyping: boolean,\n    process?: string\n  ): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const el = await this.localDesktop?.locator(selector).first(timeoutMs);\n      if (!el) throw new Error(`Element not found: ${selector}`);\n      el.typeText(text, { clearBeforeTyping });\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const parsed = this.parseProcessScopedSelector(selector, process);\n      await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"type_into_element\", {\n        process: parsed.process,\n        selector: parsed.selector,\n        text_to_type: text,\n        timeout_ms: timeoutMs,\n        clear_before_typing: clearBeforeTyping,\n        highlight_before_action: false,\n        ui_diff_before_after: false,\n      });\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async getWindowTree(process: string, title?: string, maxDepth?: number): Promise<string> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const tree = this.localDesktop?.getWindowTree(process, title, undefined);\n      return JSON.stringify(tree ?? null, null, 2);\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const toolArgs: JsonObject = {\n        process,\n        include_tree_after_action: true,\n      };\n      if (title !== undefined) toolArgs.title = title;\n      if (maxDepth !== undefined) toolArgs.tree_max_depth = maxDepth;\n      const res = await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"get_window_tree\",\n        toolArgs\n      );\n      const texts = res.content\n        .filter((c) => c.type === \"text\")\n        .map((c) => c.text)\n        .filter((t) => typeof t === \"string\");\n      return texts.join(\"\\n\");\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async clickCoordinates(\n    x: number,\n    y: number,\n    button: string = \"left\",\n    numClicks: number = 1\n  ): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      this.localDesktop?.clickAtCoordinates(x, y);\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"click_at_coordinates\",\n        { x, y, button, num_clicks: numClicks }\n      );\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async pressKey(keys: string): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      this.localDesktop?.pressKey(keys);\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"press_key\",\n        { key: keys }\n      );\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async scroll(\n    direction: string = \"down\",\n    amount: number = 3,\n    x?: number,\n    y?: number\n  ): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      // Local backend: click at position first if specified, then scroll via key presses\n      if (x !== undefined && y !== undefined) {\n        this.localDesktop?.clickAtCoordinates(x, y);\n      }\n      const scrollKey = direction === \"up\" ? \"scrollup\" : \"scrolldown\";\n      for (let i = 0; i < amount; i++) {\n        this.localDesktop?.pressKey(scrollKey);\n      }\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const toolArgs: JsonObject = { direction, amount };\n      if (x !== undefined) toolArgs.x = x;\n      if (y !== undefined) toolArgs.y = y;\n      await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"scroll\",\n        toolArgs\n      );\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async drag(\n    startX: number,\n    startY: number,\n    endX: number,\n    endY: number\n  ): Promise<void> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      // The Desktop native API does not expose a drag method.\n      // Local backend can only click the start position; full drag requires MCP.\n      logger.warn(\"[computeruse] Local drag is limited to click-at-start. Use MCP for full drag.\");\n      this.localDesktop?.clickAtCoordinates(startX, startY);\n      return;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"drag\",\n        { start_x: startX, start_y: startY, end_x: endX, end_y: endY }\n      );\n      return;\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async screenshot(): Promise<string> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const result = this.localDesktop?.captureMonitor(0);\n      if (result && typeof result === \"object\" && \"toBase64\" in result) {\n        return (result as { toBase64(): string }).toBase64();\n      }\n      return String(result ?? \"\");\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const res = await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"capture_screenshot\",\n        {}\n      );\n      const texts = res.content\n        .filter((c) => c.type === \"text\")\n        .map((c) => c.text)\n        .filter((t) => typeof t === \"string\");\n      return texts.join(\"\");\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n\n  async getApplications(): Promise<string[]> {\n    await this.ensureReady();\n    if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n    if (this.backendName === \"local\") {\n      await this.ensureLocalBackend();\n      const apps = this.localDesktop?.applications() ?? [];\n      // Normalize into human-readable names.\n      const names: string[] = [];\n      for (const app of apps) {\n        const n = app.name();\n        if (typeof n === \"string\" && n.trim().length > 0) names.push(n);\n      }\n      return names;\n    }\n\n    if (this.backendName === \"mcp\") {\n      const mcp = this.getMcp();\n      const res = await mcp.callTool(\n        this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n        \"get_applications_and_windows_list\",\n        {}\n      );\n      // Return text-only summary for now; structured parsing is done in actions/providers if needed.\n      const texts = res.content\n        .filter((c) => c.type === \"text\")\n        .map((c) => c.text)\n        .filter((t) => typeof t === \"string\");\n      return texts.length > 0 ? texts : [\"(see MCP tool output)\"];\n    }\n\n    throw new Error(\"ComputerUse backend not initialized\");\n  }\n}\n"]}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@elizaos/plugin-computeruse",
   "description": "Computer automation plugin for ElizaOS - enables AI agents to control the local machine (when supported) or a remote ComputerUse MCP server",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.8",
   "type": "module",
   "main": "dist/index.js",
   "module": "dist/index.js",