@elizaos/plugin-computeruse 2.0.0-alpha.9 → 2.0.0-beta.1

package/package.json

@@ -1,105 +1,102 @@
 {
-	"name": "@elizaos/plugin-computeruse",
-	"description": "Computer automation plugin for ElizaOS - enables AI agents to control the local machine (when supported) or a remote ComputerUse MCP server",
-	"version": "2.0.0-alpha.9",
-	"type": "module",
-	"main": "dist/index.js",
-	"module": "dist/index.js",
-	"types": "dist/index.d.ts",
-	"packageType": "plugin",
-	"platform": "node",
-	"license": "MIT",
-	"keywords": [
-		"plugin",
-		"elizaos",
-		"computeruse",
-		"automation",
-		"desktop",
-		"gui",
-		"mcp"
-	],
-	"repository": {
-		"type": "git",
-		"url": "https://github.com/elizaos/eliza"
-	},
-	"homepage": "https://elizaos.ai",
-	"exports": {
-		"./package.json": "./package.json",
-		".": {
-			"import": {
-				"types": "./dist/index.d.ts",
-				"default": "./dist/index.js"
-			}
-		}
-	},
-	"files": [
-		"dist",
-		"README.md",
-		"package.json"
-	],
-	"dependencies": {
-		"@modelcontextprotocol/sdk": "^1.21.0",
-		"zod": "^4.3.6"
-	},
-	"peerDependencies": {
-		"@elizaos/computeruse": "2.0.0-alpha.3",
-		"@elizaos/core": "2.0.0-alpha.3"
-	},
-	"peerDependenciesMeta": {
-		"@elizaos/computeruse": {
-			"optional": true
-		}
-	},
-	"devDependencies": {
-		"@biomejs/biome": "^2.3.11",
-		"@elizaos/plugin-eliza-classic": "2.0.0-alpha.3",
-		"@elizaos/plugin-inmemorydb": "2.0.0-alpha.3",
-		"@elizaos/plugin-mcp": "2.0.0-alpha.3",
-		"typescript": "^5.9.3",
-		"vitest": "^4.0.17"
-	},
-	"scripts": {
-		"dev": "bun --hot build.ts",
-		"test": "vitest run --passWithNoTests",
-		"lint": "bunx @biomejs/biome check --write --unsafe .",
-		"typecheck": "tsc --noEmit",
-		"clean": "rm -rf dist .turbo",
-		"lint:check": "bunx @biomejs/biome check .",
-		"build": "bun run build.ts",
-		"build:ts": "bun run build.ts",
-		"format": "bunx @biomejs/biome format --write .",
-		"format:check": "bunx @biomejs/biome format ."
-	},
-	"agentConfig": {
-		"pluginType": "elizaos:plugin:1.0.0",
-		"pluginParameters": {
-			"COMPUTERUSE_ENABLED": {
-				"type": "boolean",
-				"description": "Enable ComputerUse actions/providers (default: false)",
-				"required": false
-			},
-			"COMPUTERUSE_MODE": {
-				"type": "string",
-				"description": "Execution mode: auto | local | mcp (default: auto)",
-				"required": false
-			},
-			"COMPUTERUSE_MCP_SERVER": {
-				"type": "string",
-				"description": "MCP server name to use for remote control (default: computeruse)",
-				"required": false
-			}
-		}
-	},
-	"publishConfig": {
-		"access": "public"
-	},
-	"milady": {
-		"platforms": [
-			"node"
-		],
-		"runtime": "node",
-		"platformDetails": {
-			"node": "Node.js only (platform: node)"
-		}
-	}
+  "name": "@elizaos/plugin-computeruse",
+  "version": "2.0.0-beta.1",
+  "description": "Desktop automation plugin — screenshots, mouse/keyboard control, browser CDP, terminal access, file operations, and window management for elizaOS agents. Ported from open-computer-use (Apache 2.0).",
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": "./dist/index.js",
+      "default": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "package.json"
+  ],
+  "scripts": {
+    "postinstall": "node scripts/ensure-platform-deps.mjs",
+    "build": "bun run build.ts",
+    "test": "vitest run --config vitest.config.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "@elizaos/core": "2.0.0-beta.1"
+  },
+  "devDependencies": {
+    "@elizaos/core": "2.0.0-beta.1",
+    "@types/node": "^25.2.3",
+    "typescript": "^6.0.3",
+    "vitest": "4.1.5"
+  },
+  "dependencies": {
+    "@nut-tree-fork/nut-js": "^4.2.6",
+    "puppeteer-core": "^24.9.0"
+  },
+  "license": "Apache-2.0",
+  "agentConfig": {
+    "pluginType": "elizaos:plugin:1.0.0",
+    "pluginParameters": {
+      "COMPUTER_USE_ENABLED": {
+        "type": "boolean",
+        "description": "Enable computer use capabilities (desktop automation, screenshots, browser control)",
+        "required": false,
+        "default": false,
+        "sensitive": false
+      },
+      "COMPUTER_USE_SCREENSHOT_AFTER_ACTION": {
+        "type": "boolean",
+        "description": "Automatically capture a screenshot after each desktop action",
+        "required": false,
+        "default": true,
+        "sensitive": false
+      },
+      "COMPUTER_USE_ACTION_TIMEOUT_MS": {
+        "type": "number",
+        "description": "Maximum action execution timeout in milliseconds",
+        "required": false,
+        "default": 10000,
+        "sensitive": false
+      },
+      "COMPUTER_USE_APPROVAL_MODE": {
+        "type": "string",
+        "description": "Approval mode for computer-use commands: full_control auto-approves everything, smart_approve auto-approves safe read-only commands, approve_all requires approval for every command, off denies all commands.",
+        "required": false,
+        "default": "full_control",
+        "enum": [
+          "full_control",
+          "smart_approve",
+          "approve_all",
+          "off"
+        ],
+        "sensitive": false
+      },
+      "COMPUTER_USE_BROWSER_HEADLESS": {
+        "type": "boolean",
+        "description": "Run browser automation headlessly. Useful for CI and test environments without a visible desktop session.",
+        "required": false,
+        "default": false,
+        "sensitive": false
+      },
+      "ELIZA_COMPUTERUSE_DRIVER": {
+        "type": "string",
+        "description": "Cross-platform input driver: 'nutjs' (default; uses @nut-tree-fork/nut-js native bindings, works on macOS/Linux/Windows) or 'legacy' (per-OS shell tools: cliclick/xdotool/PowerShell).",
+        "required": false,
+        "default": "nutjs",
+        "enum": [
+          "nutjs",
+          "legacy"
+        ],
+        "sensitive": false
+      }
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  }
 }

package/dist/actions/click-coordinates.d.ts

@@ -1,3 +0,0 @@
-import type { Action } from "@elizaos/core";
-export declare const computeruseClickCoordinatesAction: Action;
-//# sourceMappingURL=click-coordinates.d.ts.map

package/dist/actions/click-coordinates.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"click-coordinates.d.ts","sourceRoot":"","sources":["../../src/actions/click-coordinates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,MAAM,EAQN,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,iCAAiC,EAAE,MAsL/C,CAAC"}

package/dist/actions/click-coordinates.js

@@ -1,139 +0,0 @@
-import { logger } from "@elizaos/core";
-import { getNumberParam, getStringParam } from "../utils/params.js";
-export const computeruseClickCoordinatesAction = {
-    name: "COMPUTERUSE_CLICK_COORDINATES",
-    description: "Clicks at specific screen coordinates (x, y). Use for precise clicking at known positions.",
-    similes: ["CLICK_AT", "CLICK_POSITION", "CLICK_XY"],
-    parameters: [
-        {
-            name: "x",
-            description: "X coordinate to click at",
-            required: true,
-            schema: { type: "number" },
-            examples: [500, 960],
-        },
-        {
-            name: "y",
-            description: "Y coordinate to click at",
-            required: true,
-            schema: { type: "number" },
-            examples: [300, 540],
-        },
-        {
-            name: "button",
-            description: "Mouse button: 'left', 'right', or 'middle'. Default: 'left'",
-            required: false,
-            schema: { type: "string" },
-            examples: ["left", "right"],
-        },
-        {
-            name: "numClicks",
-            description: "Number of clicks (1=single, 2=double). Default: 1",
-            required: false,
-            schema: { type: "number", default: 1, minimum: 1, maximum: 3 },
-            examples: [1, 2],
-        },
-    ],
-    validate: async (runtime, message, state, options) => {
-        const __avTextRaw = typeof message?.content?.text === "string" ? message.content.text : "";
-        const __avText = __avTextRaw.toLowerCase();
-        const __avKeywords = ["computeruse", "click", "coordinates"];
-        const __avKeywordOk = __avKeywords.length > 0 &&
-            __avKeywords.some((word) => word.length > 0 && __avText.includes(word));
-        const __avRegex = /\b(?:computeruse|click|coordinates)\b/i;
-        const __avRegexOk = __avRegex.test(__avText);
-        const __avSource = String(message?.content?.source ?? message?.source ?? "");
-        const __avExpectedSource = "";
-        const __avSourceOk = __avExpectedSource
-            ? __avSource === __avExpectedSource
-            : Boolean(__avSource ||
-                state ||
-                runtime?.agentId ||
-                runtime?.getService ||
-                runtime?.getSetting);
-        const __avOptions = options && typeof options === "object" ? options : {};
-        const __avInputOk = __avText.trim().length > 0 ||
-            Object.keys(__avOptions).length > 0 ||
-            Boolean(message?.content && typeof message.content === "object");
-        if (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {
-            return false;
-        }
-        const __avLegacyValidate = async (runtime, message, state, options) => {
-            const __avTextRaw = typeof message?.content?.text === "string" ? message.content.text : "";
-            const __avText = __avTextRaw.toLowerCase();
-            const __avKeywords = ["computeruse", "click", "coordinates"];
-            const __avKeywordOk = __avKeywords.length > 0 &&
-                __avKeywords.some((kw) => kw.length > 0 && __avText.includes(kw));
-            const __avRegex = /\b(?:computeruse|click|coordinates)\b/i;
-            const __avRegexOk = __avRegex.test(__avText);
-            const __avSource = String(message?.content?.source ?? message?.source ?? "");
-            const __avExpectedSource = "";
-            const __avSourceOk = __avExpectedSource
-                ? __avSource === __avExpectedSource
-                : Boolean(__avSource || state || runtime?.agentId || runtime?.getService);
-            const __avOptions = options && typeof options === "object" ? options : {};
-            const __avInputOk = __avText.trim().length > 0 ||
-                Object.keys(__avOptions).length > 0 ||
-                Boolean(message?.content && typeof message.content === "object");
-            if (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {
-                return false;
-            }
-            const __avLegacyValidate = async (runtime, _message) => {
-                const service = runtime.getService("computeruse");
-                return !!service && service.isEnabled();
-            };
-            try {
-                return Boolean(await __avLegacyValidate(runtime, message, state, options));
-            }
-            catch {
-                return false;
-            }
-        };
-        try {
-            return Boolean(await __avLegacyValidate(runtime, message, state, options));
-        }
-        catch {
-            return false;
-        }
-    },
-    handler: async (runtime, message, _state, options, callback) => {
-        const service = runtime.getService("computeruse");
-        if (!service) {
-            return { success: false, text: "ComputerUse service not available" };
-        }
-        const params = options?.parameters;
-        const x = getNumberParam(params, "x");
-        const y = getNumberParam(params, "y");
-        const button = getStringParam(params, "button") ?? "left";
-        const numClicks = getNumberParam(params, "numClicks") ?? 1;
-        if (x === undefined || y === undefined) {
-            return { success: false, text: "Missing required parameters: x and y" };
-        }
-        const backend = service.getBackendName();
-        try {
-            await service.clickCoordinates(x, y, button, numClicks);
-            const response = {
-                text: `Clicked at coordinates (${x}, ${y}) with ${button} button`,
-                actions: ["COMPUTERUSE_CLICK_COORDINATES"],
-                source: message.content?.source ?? "action",
-            };
-            await callback?.(response);
-            return {
-                success: true,
-                text: response.text ?? "",
-                values: { x, y, button, numClicks },
-                data: { x, y, button, numClicks, backend },
-            };
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            logger.error(`[computeruse] click_coordinates failed: ${msg}`);
-            return {
-                success: false,
-                text: `ComputerUse click at coordinates failed: ${msg}`,
-                values: { x, y, button, numClicks },
-            };
-        }
-    },
-};
-//# sourceMappingURL=click-coordinates.js.map

package/dist/actions/click-coordinates.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"click-coordinates.js","sourceRoot":"","sources":["../../src/actions/click-coordinates.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpE,MAAM,CAAC,MAAM,iCAAiC,GAAW;IACxD,IAAI,EAAE,+BAA+B;IACrC,WAAW,EACV,4FAA4F;IAC7F,OAAO,EAAE,CAAC,UAAU,EAAE,gBAAgB,EAAE,UAAU,CAAC;IACnD,UAAU,EAAE;QACX;YACC,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,0BAA0B;YACvC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB;QACD;YACC,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,0BAA0B;YACvC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB;QACD;YACC,IAAI,EAAE,QAAQ;YACd,WAAW,EACV,6DAA6D;YAC9D,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;SAC3B;QACD;YACC,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,mDAAmD;YAChE,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;YAC9D,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;SAChB;KACD;IAED,QAAQ,EAAE,KAAK,EACd,OAAY,EACZ,OAAY,EACZ,KAAW,EACX,OAAa,EACM,EAAE;QACrB,MAAM,WAAW,GAChB,OAAO,OAAO,EAAE,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QAC7D,MAAM,aAAa,GAClB,YAAY,CAAC,MAAM,GAAG,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,wCAAwC,CAAC;QAC3D,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,MAAM,CACxB,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,OAAO,EAAE,MAAM,IAAI,EAAE,CACjD,CAAC;QACF,MAAM,kBAAkB,GAAG,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,kBAAkB;YACtC,CAAC,CAAC,UAAU,KAAK,kBAAkB;YACnC,CAAC,CAAC,OAAO,CACP,UAAU;gBACT,KAAK;gBACL,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,UAAU;gBACnB,OAAO,EAAE,UAAU,CACpB,CAAC;QACJ,MAAM,WAAW,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,WAAW,GAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,WAAsC,CAAC,CAAC,MAAM,GAAG,CAAC;YAC9D,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAElE,IAAI,CAAC,CAAC,aAAa,IAAI,WAAW,IAAI,YAAY,IAAI,WAAW,CAAC,EAAE,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QAED,MAAM,kBAAkB,GAAG,KAAK,EAC/B,OAAY,EACZ,OAAY,EACZ,KAAW,EACX,OAAa,EACM,EAAE;YACrB,MAAM,WAAW,GAChB,OAAO,OAAO,EAAE,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;YAC7D,MAAM,aAAa,GAClB,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,wCAAwC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,UAAU,GAAG,MAAM,CACxB,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,OAAO,EAAE,MAAM,IAAI,EAAE,CACjD,CAAC;YACF,MAAM,kBAAkB,GAAG,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,kBAAkB;gBACtC,CAAC,CAAC,UAAU,KAAK,kBAAkB;gBACnC,CAAC,CAAC,OAAO,CACP,UAAU,IAAI,KAAK,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,UAAU,CAC9D,CAAC;YACJ,MAAM,WAAW,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1E,MAAM,WAAW,GAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,WAAsC,CAAC,CAAC,MAAM,GAAG,CAAC;gBAC9D,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;YAElE,IAAI,CAAC,CAAC,aAAa,IAAI,WAAW,IAAI,YAAY,IAAI,WAAW,CAAC,EAAE,CAAC;gBACpE,OAAO,KAAK,CAAC;YACd,CAAC;YAED,MAAM,kBAAkB,GAAG,KAAK,EAC/B,OAAsB,EACtB,QAAgB,EACG,EAAE;gBACrB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;gBACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACzC,CAAC,CAAC;YACF,IAAI,CAAC;gBACJ,OAAO,OAAO,CACb,MAAO,kBAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACnE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,KAAK,CAAC;YACd,CAAC;QACF,CAAC,CAAC;QACF,IAAI,CAAC;YACJ,OAAO,OAAO,CACb,MAAO,kBAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACnE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EACb,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACF,EAAE;QAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC;QAC1D,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACxC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,sCAAsC,EAAE,CAAC;QACzE,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACJ,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;YAExD,MAAM,QAAQ,GAAY;gBACzB,IAAI,EAAE,2BAA2B,CAAC,KAAK,CAAC,UAAU,MAAM,SAAS;gBACjE,OAAO,EAAE,CAAC,+BAA+B,CAAC;gBAC1C,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC3C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;gBACnC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE;aAC1C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;YAC/D,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,4CAA4C,GAAG,EAAE;gBACvD,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;aACnC,CAAC;QACH,CAAC;IACF,CAAC;CACD,CAAC","sourcesContent":["import type {\n\tAction,\n\tActionResult,\n\tContent,\n\tHandlerCallback,\n\tHandlerOptions,\n\tIAgentRuntime,\n\tMemory,\n\tState,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getNumberParam, getStringParam } from \"../utils/params.js\";\n\nexport const computeruseClickCoordinatesAction: Action = {\n\tname: \"COMPUTERUSE_CLICK_COORDINATES\",\n\tdescription:\n\t\t\"Clicks at specific screen coordinates (x, y). Use for precise clicking at known positions.\",\n\tsimiles: [\"CLICK_AT\", \"CLICK_POSITION\", \"CLICK_XY\"],\n\tparameters: [\n\t\t{\n\t\t\tname: \"x\",\n\t\t\tdescription: \"X coordinate to click at\",\n\t\t\trequired: true,\n\t\t\tschema: { type: \"number\" },\n\t\t\texamples: [500, 960],\n\t\t},\n\t\t{\n\t\t\tname: \"y\",\n\t\t\tdescription: \"Y coordinate to click at\",\n\t\t\trequired: true,\n\t\t\tschema: { type: \"number\" },\n\t\t\texamples: [300, 540],\n\t\t},\n\t\t{\n\t\t\tname: \"button\",\n\t\t\tdescription:\n\t\t\t\t\"Mouse button: 'left', 'right', or 'middle'. Default: 'left'\",\n\t\t\trequired: false,\n\t\t\tschema: { type: \"string\" },\n\t\t\texamples: [\"left\", \"right\"],\n\t\t},\n\t\t{\n\t\t\tname: \"numClicks\",\n\t\t\tdescription: \"Number of clicks (1=single, 2=double). Default: 1\",\n\t\t\trequired: false,\n\t\t\tschema: { type: \"number\", default: 1, minimum: 1, maximum: 3 },\n\t\t\texamples: [1, 2],\n\t\t},\n\t],\n\n\tvalidate: async (\n\t\truntime: any,\n\t\tmessage: any,\n\t\tstate?: any,\n\t\toptions?: any,\n\t): Promise<boolean> => {\n\t\tconst __avTextRaw =\n\t\t\ttypeof message?.content?.text === \"string\" ? message.content.text : \"\";\n\t\tconst __avText = __avTextRaw.toLowerCase();\n\t\tconst __avKeywords = [\"computeruse\", \"click\", \"coordinates\"];\n\t\tconst __avKeywordOk =\n\t\t\t__avKeywords.length > 0 &&\n\t\t\t__avKeywords.some((word) => word.length > 0 && __avText.includes(word));\n\t\tconst __avRegex = /\\b(?:computeruse|click|coordinates)\\b/i;\n\t\tconst __avRegexOk = __avRegex.test(__avText);\n\t\tconst __avSource = String(\n\t\t\tmessage?.content?.source ?? message?.source ?? \"\",\n\t\t);\n\t\tconst __avExpectedSource = \"\";\n\t\tconst __avSourceOk = __avExpectedSource\n\t\t\t? __avSource === __avExpectedSource\n\t\t\t: Boolean(\n\t\t\t\t\t__avSource ||\n\t\t\t\t\t\tstate ||\n\t\t\t\t\t\truntime?.agentId ||\n\t\t\t\t\t\truntime?.getService ||\n\t\t\t\t\t\truntime?.getSetting,\n\t\t\t\t);\n\t\tconst __avOptions = options && typeof options === \"object\" ? options : {};\n\t\tconst __avInputOk =\n\t\t\t__avText.trim().length > 0 ||\n\t\t\tObject.keys(__avOptions as Record<string, unknown>).length > 0 ||\n\t\t\tBoolean(message?.content && typeof message.content === \"object\");\n\n\t\tif (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {\n\t\t\treturn false;\n\t\t}\n\n\t\tconst __avLegacyValidate = async (\n\t\t\truntime: any,\n\t\t\tmessage: any,\n\t\t\tstate?: any,\n\t\t\toptions?: any,\n\t\t): Promise<boolean> => {\n\t\t\tconst __avTextRaw =\n\t\t\t\ttypeof message?.content?.text === \"string\" ? message.content.text : \"\";\n\t\t\tconst __avText = __avTextRaw.toLowerCase();\n\t\t\tconst __avKeywords = [\"computeruse\", \"click\", \"coordinates\"];\n\t\t\tconst __avKeywordOk =\n\t\t\t\t__avKeywords.length > 0 &&\n\t\t\t\t__avKeywords.some((kw) => kw.length > 0 && __avText.includes(kw));\n\t\t\tconst __avRegex = /\\b(?:computeruse|click|coordinates)\\b/i;\n\t\t\tconst __avRegexOk = __avRegex.test(__avText);\n\t\t\tconst __avSource = String(\n\t\t\t\tmessage?.content?.source ?? message?.source ?? \"\",\n\t\t\t);\n\t\t\tconst __avExpectedSource = \"\";\n\t\t\tconst __avSourceOk = __avExpectedSource\n\t\t\t\t? __avSource === __avExpectedSource\n\t\t\t\t: Boolean(\n\t\t\t\t\t\t__avSource || state || runtime?.agentId || runtime?.getService,\n\t\t\t\t\t);\n\t\t\tconst __avOptions = options && typeof options === \"object\" ? options : {};\n\t\t\tconst __avInputOk =\n\t\t\t\t__avText.trim().length > 0 ||\n\t\t\t\tObject.keys(__avOptions as Record<string, unknown>).length > 0 ||\n\t\t\t\tBoolean(message?.content && typeof message.content === \"object\");\n\n\t\t\tif (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {\n\t\t\t\treturn false;\n\t\t\t}\n\n\t\t\tconst __avLegacyValidate = async (\n\t\t\t\truntime: IAgentRuntime,\n\t\t\t\t_message: Memory,\n\t\t\t): Promise<boolean> => {\n\t\t\t\tconst service = runtime.getService<ComputerUseService>(\"computeruse\");\n\t\t\t\treturn !!service && service.isEnabled();\n\t\t\t};\n\t\t\ttry {\n\t\t\t\treturn Boolean(\n\t\t\t\t\tawait (__avLegacyValidate as any)(runtime, message, state, options),\n\t\t\t\t);\n\t\t\t} catch {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t};\n\t\ttry {\n\t\t\treturn Boolean(\n\t\t\t\tawait (__avLegacyValidate as any)(runtime, message, state, options),\n\t\t\t);\n\t\t} catch {\n\t\t\treturn false;\n\t\t}\n\t},\n\n\thandler: async (\n\t\truntime: IAgentRuntime,\n\t\tmessage: Memory,\n\t\t_state?: State,\n\t\toptions?: HandlerOptions,\n\t\tcallback?: HandlerCallback,\n\t): Promise<ActionResult> => {\n\t\tconst service = runtime.getService<ComputerUseService>(\"computeruse\");\n\t\tif (!service) {\n\t\t\treturn { success: false, text: \"ComputerUse service not available\" };\n\t\t}\n\n\t\tconst params = options?.parameters;\n\t\tconst x = getNumberParam(params, \"x\");\n\t\tconst y = getNumberParam(params, \"y\");\n\t\tconst button = getStringParam(params, \"button\") ?? \"left\";\n\t\tconst numClicks = getNumberParam(params, \"numClicks\") ?? 1;\n\n\t\tif (x === undefined || y === undefined) {\n\t\t\treturn { success: false, text: \"Missing required parameters: x and y\" };\n\t\t}\n\n\t\tconst backend = service.getBackendName();\n\n\t\ttry {\n\t\t\tawait service.clickCoordinates(x, y, button, numClicks);\n\n\t\t\tconst response: Content = {\n\t\t\t\ttext: `Clicked at coordinates (${x}, ${y}) with ${button} button`,\n\t\t\t\tactions: [\"COMPUTERUSE_CLICK_COORDINATES\"],\n\t\t\t\tsource: message.content?.source ?? \"action\",\n\t\t\t};\n\t\t\tawait callback?.(response);\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\ttext: response.text ?? \"\",\n\t\t\t\tvalues: { x, y, button, numClicks },\n\t\t\t\tdata: { x, y, button, numClicks, backend },\n\t\t\t};\n\t\t} catch (err) {\n\t\t\tconst msg = err instanceof Error ? err.message : String(err);\n\t\t\tlogger.error(`[computeruse] click_coordinates failed: ${msg}`);\n\t\t\treturn {\n\t\t\t\tsuccess: false,\n\t\t\t\ttext: `ComputerUse click at coordinates failed: ${msg}`,\n\t\t\t\tvalues: { x, y, button, numClicks },\n\t\t\t};\n\t\t}\n\t},\n};\n"]}

package/dist/actions/click.d.ts

@@ -1,3 +0,0 @@
-import type { Action } from "@elizaos/core";
-export declare const computeruseClickAction: Action;
-//# sourceMappingURL=click.d.ts.map

package/dist/actions/click.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,MAAM,EAQN,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,sBAAsB,EAAE,MAwLpC,CAAC"}

package/dist/actions/click.js

@@ -1,144 +0,0 @@
-import { logger } from "@elizaos/core";
-import { getNumberParam, getStringParam } from "../utils/params.js";
-export const computeruseClickAction = {
-    name: "COMPUTERUSE_CLICK",
-    description: "Clicks a UI element on the computer using a ComputerUse selector.",
-    similes: ["CLICK_UI", "CLICK_ELEMENT", "TAP_UI"],
-    parameters: [
-        {
-            name: "process",
-            description: "Process name to scope the click when using MCP mode (e.g. 'chrome', 'notepad'). Optional if selector is prefixed with 'process:<name> >> ...'.",
-            required: false,
-            schema: { type: "string" },
-            examples: ["chrome", "notepad"],
-        },
-        {
-            name: "selector",
-            description: "ComputerUse selector string for the target element",
-            required: true,
-            schema: { type: "string" },
-            examples: ["role:Button && name:Submit", "role:Edit && name:Search"],
-        },
-        {
-            name: "timeoutMs",
-            description: "Timeout in milliseconds to find the element",
-            required: false,
-            schema: { type: "number", default: 5000, minimum: 0 },
-            examples: [5000],
-        },
-    ],
-    validate: async (runtime, message, state, options) => {
-        const __avTextRaw = typeof message?.content?.text === "string" ? message.content.text : "";
-        const __avText = __avTextRaw.toLowerCase();
-        const __avKeywords = ["computeruse", "click"];
-        const __avKeywordOk = __avKeywords.length > 0 &&
-            __avKeywords.some((word) => word.length > 0 && __avText.includes(word));
-        const __avRegex = /\b(?:computeruse|click)\b/i;
-        const __avRegexOk = __avRegex.test(__avText);
-        const __avSource = String(message?.content?.source ?? message?.source ?? "");
-        const __avExpectedSource = "";
-        const __avSourceOk = __avExpectedSource
-            ? __avSource === __avExpectedSource
-            : Boolean(__avSource ||
-                state ||
-                runtime?.agentId ||
-                runtime?.getService ||
-                runtime?.getSetting);
-        const __avOptions = options && typeof options === "object" ? options : {};
-        const __avInputOk = __avText.trim().length > 0 ||
-            Object.keys(__avOptions).length > 0 ||
-            Boolean(message?.content && typeof message.content === "object");
-        if (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {
-            return false;
-        }
-        const __avLegacyValidate = async (runtime, message, state, options) => {
-            const __avTextRaw = typeof message?.content?.text === "string" ? message.content.text : "";
-            const __avText = __avTextRaw.toLowerCase();
-            const __avKeywords = ["computeruse", "click"];
-            const __avKeywordOk = __avKeywords.length > 0 &&
-                __avKeywords.some((kw) => kw.length > 0 && __avText.includes(kw));
-            const __avRegex = /\b(?:computeruse|click)\b/i;
-            const __avRegexOk = __avRegex.test(__avText);
-            const __avSource = String(message?.content?.source ?? message?.source ?? "");
-            const __avExpectedSource = "";
-            const __avSourceOk = __avExpectedSource
-                ? __avSource === __avExpectedSource
-                : Boolean(__avSource || state || runtime?.agentId || runtime?.getService);
-            const __avOptions = options && typeof options === "object" ? options : {};
-            const __avInputOk = __avText.trim().length > 0 ||
-                Object.keys(__avOptions).length > 0 ||
-                Boolean(message?.content && typeof message.content === "object");
-            if (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {
-                return false;
-            }
-            const __avLegacyValidate = async (runtime, message) => {
-                const service = runtime.getService("computeruse");
-                if (!service || !service.isEnabled())
-                    return false;
-                const text = message.content?.text?.toLowerCase() ?? "";
-                return text.includes("click") || text.includes("tap");
-            };
-            try {
-                return Boolean(await __avLegacyValidate(runtime, message, state, options));
-            }
-            catch {
-                return false;
-            }
-        };
-        try {
-            return Boolean(await __avLegacyValidate(runtime, message, state, options));
-        }
-        catch {
-            return false;
-        }
-    },
-    handler: async (runtime, message, _state, options, callback) => {
-        const service = runtime.getService("computeruse");
-        if (!service) {
-            return { success: false, text: "ComputerUse service not available" };
-        }
-        const params = options?.parameters;
-        const process = getStringParam(params, "process");
-        const selector = getStringParam(params, "selector");
-        const timeoutMs = getNumberParam(params, "timeoutMs") ?? 5000;
-        if (!selector) {
-            return { success: false, text: "Missing required parameter: selector" };
-        }
-        try {
-            await service.click(selector, timeoutMs, process);
-            const response = {
-                text: `Clicked element: ${selector}`,
-                actions: ["COMPUTERUSE_CLICK"],
-                source: message.content?.source ?? "action",
-            };
-            await callback?.(response);
-            return {
-                success: true,
-                text: response.text ?? "",
-                values: { process, selector, timeoutMs },
-                data: {
-                    process,
-                    selector,
-                    timeoutMs,
-                    backend: service.getBackendName(),
-                },
-            };
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            logger.error(`[computeruse] click failed: ${msg}`);
-            return {
-                success: false,
-                text: `ComputerUse click failed: ${msg}`,
-                values: { process, selector, timeoutMs },
-                data: {
-                    process,
-                    selector,
-                    timeoutMs,
-                    backend: service.getBackendName(),
-                },
-            };
-        }
-    },
-};
-//# sourceMappingURL=click.js.map

package/dist/actions/click.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpE,MAAM,CAAC,MAAM,sBAAsB,GAAW;IAC7C,IAAI,EAAE,mBAAmB;IACzB,WAAW,EACV,mEAAmE;IACpE,OAAO,EAAE,CAAC,UAAU,EAAE,eAAe,EAAE,QAAQ,CAAC;IAChD,UAAU,EAAE;QACX;YACC,IAAI,EAAE,SAAS;YACf,WAAW,EACV,gJAAgJ;YACjJ,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC/B;QACD;YACC,IAAI,EAAE,UAAU;YAChB,WAAW,EAAE,oDAAoD;YACjE,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,4BAA4B,EAAE,0BAA0B,CAAC;SACpE;QACD;YACC,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,6CAA6C;YAC1D,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE;YACrD,QAAQ,EAAE,CAAC,IAAI,CAAC;SAChB;KACD;IAED,QAAQ,EAAE,KAAK,EACd,OAAY,EACZ,OAAY,EACZ,KAAW,EACX,OAAa,EACM,EAAE;QACrB,MAAM,WAAW,GAChB,OAAO,OAAO,EAAE,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,aAAa,GAClB,YAAY,CAAC,MAAM,GAAG,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,4BAA4B,CAAC;QAC/C,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,MAAM,CACxB,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,OAAO,EAAE,MAAM,IAAI,EAAE,CACjD,CAAC;QACF,MAAM,kBAAkB,GAAG,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,kBAAkB;YACtC,CAAC,CAAC,UAAU,KAAK,kBAAkB;YACnC,CAAC,CAAC,OAAO,CACP,UAAU;gBACT,KAAK;gBACL,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,UAAU;gBACnB,OAAO,EAAE,UAAU,CACpB,CAAC;QACJ,MAAM,WAAW,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,WAAW,GAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,WAAsC,CAAC,CAAC,MAAM,GAAG,CAAC;YAC9D,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAElE,IAAI,CAAC,CAAC,aAAa,IAAI,WAAW,IAAI,YAAY,IAAI,WAAW,CAAC,EAAE,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QAED,MAAM,kBAAkB,GAAG,KAAK,EAC/B,OAAY,EACZ,OAAY,EACZ,KAAW,EACX,OAAa,EACM,EAAE;YACrB,MAAM,WAAW,GAChB,OAAO,OAAO,EAAE,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;YAC9C,MAAM,aAAa,GAClB,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,4BAA4B,CAAC;YAC/C,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,UAAU,GAAG,MAAM,CACxB,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,OAAO,EAAE,MAAM,IAAI,EAAE,CACjD,CAAC;YACF,MAAM,kBAAkB,GAAG,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,kBAAkB;gBACtC,CAAC,CAAC,UAAU,KAAK,kBAAkB;gBACnC,CAAC,CAAC,OAAO,CACP,UAAU,IAAI,KAAK,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,UAAU,CAC9D,CAAC;YACJ,MAAM,WAAW,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1E,MAAM,WAAW,GAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,WAAsC,CAAC,CAAC,MAAM,GAAG,CAAC;gBAC9D,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;YAElE,IAAI,CAAC,CAAC,aAAa,IAAI,WAAW,IAAI,YAAY,IAAI,WAAW,CAAC,EAAE,CAAC;gBACpE,OAAO,KAAK,CAAC;YACd,CAAC;YAED,MAAM,kBAAkB,GAAG,KAAK,EAC/B,OAAsB,EACtB,OAAe,EACI,EAAE;gBACrB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;gBACtE,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBAAE,OAAO,KAAK,CAAC;gBAEnD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACvD,CAAC,CAAC;YACF,IAAI,CAAC;gBACJ,OAAO,OAAO,CACb,MAAO,kBAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACnE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,KAAK,CAAC;YACd,CAAC;QACF,CAAC,CAAC;QACF,IAAI,CAAC;YACJ,OAAO,OAAO,CACb,MAAO,kBAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACnE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EACb,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACF,EAAE;QAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,IAAI,CAAC;QAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,sCAAsC,EAAE,CAAC;QACzE,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAY;gBACzB,IAAI,EAAE,oBAAoB,QAAQ,EAAE;gBACpC,OAAO,EAAE,CAAC,mBAAmB,CAAC;gBAC9B,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC3C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE;gBACxC,IAAI,EAAE;oBACL,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE;iBACjC;aACD,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;YACnD,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,6BAA6B,GAAG,EAAE;gBACxC,MAAM,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE;gBACxC,IAAI,EAAE;oBACL,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE;iBACjC;aACD,CAAC;QACH,CAAC;IACF,CAAC;CACD,CAAC","sourcesContent":["import type {\n\tAction,\n\tActionResult,\n\tContent,\n\tHandlerCallback,\n\tHandlerOptions,\n\tIAgentRuntime,\n\tMemory,\n\tState,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getNumberParam, getStringParam } from \"../utils/params.js\";\n\nexport const computeruseClickAction: Action = {\n\tname: \"COMPUTERUSE_CLICK\",\n\tdescription:\n\t\t\"Clicks a UI element on the computer using a ComputerUse selector.\",\n\tsimiles: [\"CLICK_UI\", \"CLICK_ELEMENT\", \"TAP_UI\"],\n\tparameters: [\n\t\t{\n\t\t\tname: \"process\",\n\t\t\tdescription:\n\t\t\t\t\"Process name to scope the click when using MCP mode (e.g. 'chrome', 'notepad'). Optional if selector is prefixed with 'process:<name> >> ...'.\",\n\t\t\trequired: false,\n\t\t\tschema: { type: \"string\" },\n\t\t\texamples: [\"chrome\", \"notepad\"],\n\t\t},\n\t\t{\n\t\t\tname: \"selector\",\n\t\t\tdescription: \"ComputerUse selector string for the target element\",\n\t\t\trequired: true,\n\t\t\tschema: { type: \"string\" },\n\t\t\texamples: [\"role:Button && name:Submit\", \"role:Edit && name:Search\"],\n\t\t},\n\t\t{\n\t\t\tname: \"timeoutMs\",\n\t\t\tdescription: \"Timeout in milliseconds to find the element\",\n\t\t\trequired: false,\n\t\t\tschema: { type: \"number\", default: 5000, minimum: 0 },\n\t\t\texamples: [5000],\n\t\t},\n\t],\n\n\tvalidate: async (\n\t\truntime: any,\n\t\tmessage: any,\n\t\tstate?: any,\n\t\toptions?: any,\n\t): Promise<boolean> => {\n\t\tconst __avTextRaw =\n\t\t\ttypeof message?.content?.text === \"string\" ? message.content.text : \"\";\n\t\tconst __avText = __avTextRaw.toLowerCase();\n\t\tconst __avKeywords = [\"computeruse\", \"click\"];\n\t\tconst __avKeywordOk =\n\t\t\t__avKeywords.length > 0 &&\n\t\t\t__avKeywords.some((word) => word.length > 0 && __avText.includes(word));\n\t\tconst __avRegex = /\\b(?:computeruse|click)\\b/i;\n\t\tconst __avRegexOk = __avRegex.test(__avText);\n\t\tconst __avSource = String(\n\t\t\tmessage?.content?.source ?? message?.source ?? \"\",\n\t\t);\n\t\tconst __avExpectedSource = \"\";\n\t\tconst __avSourceOk = __avExpectedSource\n\t\t\t? __avSource === __avExpectedSource\n\t\t\t: Boolean(\n\t\t\t\t\t__avSource ||\n\t\t\t\t\t\tstate ||\n\t\t\t\t\t\truntime?.agentId ||\n\t\t\t\t\t\truntime?.getService ||\n\t\t\t\t\t\truntime?.getSetting,\n\t\t\t\t);\n\t\tconst __avOptions = options && typeof options === \"object\" ? options : {};\n\t\tconst __avInputOk =\n\t\t\t__avText.trim().length > 0 ||\n\t\t\tObject.keys(__avOptions as Record<string, unknown>).length > 0 ||\n\t\t\tBoolean(message?.content && typeof message.content === \"object\");\n\n\t\tif (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {\n\t\t\treturn false;\n\t\t}\n\n\t\tconst __avLegacyValidate = async (\n\t\t\truntime: any,\n\t\t\tmessage: any,\n\t\t\tstate?: any,\n\t\t\toptions?: any,\n\t\t): Promise<boolean> => {\n\t\t\tconst __avTextRaw =\n\t\t\t\ttypeof message?.content?.text === \"string\" ? message.content.text : \"\";\n\t\t\tconst __avText = __avTextRaw.toLowerCase();\n\t\t\tconst __avKeywords = [\"computeruse\", \"click\"];\n\t\t\tconst __avKeywordOk =\n\t\t\t\t__avKeywords.length > 0 &&\n\t\t\t\t__avKeywords.some((kw) => kw.length > 0 && __avText.includes(kw));\n\t\t\tconst __avRegex = /\\b(?:computeruse|click)\\b/i;\n\t\t\tconst __avRegexOk = __avRegex.test(__avText);\n\t\t\tconst __avSource = String(\n\t\t\t\tmessage?.content?.source ?? message?.source ?? \"\",\n\t\t\t);\n\t\t\tconst __avExpectedSource = \"\";\n\t\t\tconst __avSourceOk = __avExpectedSource\n\t\t\t\t? __avSource === __avExpectedSource\n\t\t\t\t: Boolean(\n\t\t\t\t\t\t__avSource || state || runtime?.agentId || runtime?.getService,\n\t\t\t\t\t);\n\t\t\tconst __avOptions = options && typeof options === \"object\" ? options : {};\n\t\t\tconst __avInputOk =\n\t\t\t\t__avText.trim().length > 0 ||\n\t\t\t\tObject.keys(__avOptions as Record<string, unknown>).length > 0 ||\n\t\t\t\tBoolean(message?.content && typeof message.content === \"object\");\n\n\t\t\tif (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {\n\t\t\t\treturn false;\n\t\t\t}\n\n\t\t\tconst __avLegacyValidate = async (\n\t\t\t\truntime: IAgentRuntime,\n\t\t\t\tmessage: Memory,\n\t\t\t): Promise<boolean> => {\n\t\t\t\tconst service = runtime.getService<ComputerUseService>(\"computeruse\");\n\t\t\t\tif (!service || !service.isEnabled()) return false;\n\n\t\t\t\tconst text = message.content?.text?.toLowerCase() ?? \"\";\n\t\t\t\treturn text.includes(\"click\") || text.includes(\"tap\");\n\t\t\t};\n\t\t\ttry {\n\t\t\t\treturn Boolean(\n\t\t\t\t\tawait (__avLegacyValidate as any)(runtime, message, state, options),\n\t\t\t\t);\n\t\t\t} catch {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t};\n\t\ttry {\n\t\t\treturn Boolean(\n\t\t\t\tawait (__avLegacyValidate as any)(runtime, message, state, options),\n\t\t\t);\n\t\t} catch {\n\t\t\treturn false;\n\t\t}\n\t},\n\n\thandler: async (\n\t\truntime: IAgentRuntime,\n\t\tmessage: Memory,\n\t\t_state?: State,\n\t\toptions?: HandlerOptions,\n\t\tcallback?: HandlerCallback,\n\t): Promise<ActionResult> => {\n\t\tconst service = runtime.getService<ComputerUseService>(\"computeruse\");\n\t\tif (!service) {\n\t\t\treturn { success: false, text: \"ComputerUse service not available\" };\n\t\t}\n\n\t\tconst params = options?.parameters;\n\t\tconst process = getStringParam(params, \"process\");\n\t\tconst selector = getStringParam(params, \"selector\");\n\t\tconst timeoutMs = getNumberParam(params, \"timeoutMs\") ?? 5000;\n\t\tif (!selector) {\n\t\t\treturn { success: false, text: \"Missing required parameter: selector\" };\n\t\t}\n\n\t\ttry {\n\t\t\tawait service.click(selector, timeoutMs, process);\n\t\t\tconst response: Content = {\n\t\t\t\ttext: `Clicked element: ${selector}`,\n\t\t\t\tactions: [\"COMPUTERUSE_CLICK\"],\n\t\t\t\tsource: message.content?.source ?? \"action\",\n\t\t\t};\n\t\t\tawait callback?.(response);\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\ttext: response.text ?? \"\",\n\t\t\t\tvalues: { process, selector, timeoutMs },\n\t\t\t\tdata: {\n\t\t\t\t\tprocess,\n\t\t\t\t\tselector,\n\t\t\t\t\ttimeoutMs,\n\t\t\t\t\tbackend: service.getBackendName(),\n\t\t\t\t},\n\t\t\t};\n\t\t} catch (err) {\n\t\t\tconst msg = err instanceof Error ? err.message : String(err);\n\t\t\tlogger.error(`[computeruse] click failed: ${msg}`);\n\t\t\treturn {\n\t\t\t\tsuccess: false,\n\t\t\t\ttext: `ComputerUse click failed: ${msg}`,\n\t\t\t\tvalues: { process, selector, timeoutMs },\n\t\t\t\tdata: {\n\t\t\t\t\tprocess,\n\t\t\t\t\tselector,\n\t\t\t\t\ttimeoutMs,\n\t\t\t\t\tbackend: service.getBackendName(),\n\t\t\t\t},\n\t\t\t};\n\t\t}\n\t},\n};\n"]}

package/dist/actions/drag.d.ts

@@ -1,3 +0,0 @@
-import type { Action } from "@elizaos/core";
-export declare const computeruseDragAction: Action;
-//# sourceMappingURL=drag.d.ts.map

package/dist/actions/drag.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"drag.d.ts","sourceRoot":"","sources":["../../src/actions/drag.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,MAAM,EAQN,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,qBAAqB,EAAE,MA6LnC,CAAC"}