@elizaos/plugin-computeruse 2.0.0-alpha.9 → 2.0.11-beta.7

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shaw Walters and elizaOS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,80 @@
+# `@elizaos/plugin-computeruse`
+
+Desktop automation plugin for elizaOS agents — screenshots, mouse /
+keyboard control, browser CDP automation, window management, clipboard,
+and the OCR provider registry that other plugins contribute to.
+
+Ported from
+[`coasty-ai/open-computer-use`](https://github.com/coasty-ai/open-computer-use)
+(Apache 2.0).
+
+## Boundary with `@elizaos/plugin-vision`
+
+This plugin owns the OS surfaces:
+
+- screen / display capture (`src/platform/capture.ts`,
+  `src/platform/displays.ts`,
+  `ComputerUseService.captureScreen()`),
+- input + windows + clipboard + accessibility,
+- the OCR provider registries — `OcrProvider` (line-level) and
+  `CoordOcrProvider` (hierarchical with absolute coords), defined in
+  `src/mobile/ocr-provider.ts`.
+
+`@elizaos/plugin-vision` owns the camera pipeline, scene description
+via `runtime.useModel(IMAGE_DESCRIPTION)`, the screen tiler, the
+detector pipeline (faces / people / objects), and the OCR
+implementations themselves. plugin-vision *consumes* capture from this
+plugin via `runtime.getService("computeruse")` and *contributes* the
+hierarchical OCR adapter into this plugin's `registerCoordOcrProvider`
+seam at boot.
+
+Both seams are runtime feature-detected — neither package depends on
+the other.
+
+## Enabling
+
+- Config: `features.computeruse: true`
+- Env: `COMPUTER_USE_ENABLED=1`
+
+## Platform requirements
+
+| OS | Capture | Input |
+|----|---------|-------|
+| macOS | `screencapture` (built-in) | `cliclick` (`brew install cliclick`), AppleScript |
+| Linux | `import` (ImageMagick) / `scrot` | `xdotool` (`sudo apt install xdotool`) |
+| Windows | PowerShell + `System.Drawing` | PowerShell |
+| Browser | — | `puppeteer-core` + Chrome / Edge / Brave |
+
+## Surface
+
+- **Actions** — `COMPUTER_USE` (canonical screenshot / click / key /
+  scroll / etc.), `WINDOW` (list / focus / arrange / move /...), and
+  `COMPUTER_USE_AGENT` (high-level goal-driven autonomous desktop loop:
+  Brain → Cascade → dispatch up to `maxSteps` iterations).
+  Subactions of `COMPUTER_USE` and `WINDOW` are promoted to virtual
+  top-level actions (e.g. `COMPUTER_USE_CLICK`, `WINDOW_FOCUS`) so the
+  planner picks a specific verb directly from the catalogue.
+- **Services** — `ComputerUseService` (`serviceType = "computeruse"`)
+  and `VisionContextProvider`.
+- **Providers** — `computerStateProvider`, `sceneProvider`.
+- **Routes** — approval inbox + SSE stream + approval-mode toggle under
+  `/api/computer-use/...`.
+
+## File operations + shell
+
+File operations live on the FILE action; shell / terminal access lives
+on the SHELL action. They are **not** exposed by this plugin.
+
+## Further reading
+
+- [`docs/MULTI_MONITOR.md`](./docs/MULTI_MONITOR.md) — multi-display
+  capture and coordinate translation.
+- [`docs/SCENE_BUILDER.md`](./docs/SCENE_BUILDER.md) — how windows,
+  a11y, screen, and OCR are composed into a single `Scene`.
+- [`docs/IOS_CONSTRAINTS.md`](./docs/IOS_CONSTRAINTS.md) /
+  [`docs/ANDROID_CONSTRAINTS.md`](./docs/ANDROID_CONSTRAINTS.md) —
+  honest scope on mobile.
+- [`docs/MOBILE_ASSISTANT_ROUTING.md`](./docs/MOBILE_ASSISTANT_ROUTING.md)
+  — mobile request routing.
+- [`docs/AOSP_SYSTEM_APP.md`](./docs/AOSP_SYSTEM_APP.md) — AOSP
+  system-app deployment notes.

package/package.json

@@ -1,105 +1,128 @@
 {
-	"name": "@elizaos/plugin-computeruse",
-	"description": "Computer automation plugin for ElizaOS - enables AI agents to control the local machine (when supported) or a remote ComputerUse MCP server",
-	"version": "2.0.0-alpha.9",
-	"type": "module",
-	"main": "dist/index.js",
-	"module": "dist/index.js",
-	"types": "dist/index.d.ts",
-	"packageType": "plugin",
-	"platform": "node",
-	"license": "MIT",
-	"keywords": [
-		"plugin",
-		"elizaos",
-		"computeruse",
-		"automation",
-		"desktop",
-		"gui",
-		"mcp"
-	],
-	"repository": {
-		"type": "git",
-		"url": "https://github.com/elizaos/eliza"
-	},
-	"homepage": "https://elizaos.ai",
-	"exports": {
-		"./package.json": "./package.json",
-		".": {
-			"import": {
-				"types": "./dist/index.d.ts",
-				"default": "./dist/index.js"
-			}
-		}
-	},
-	"files": [
-		"dist",
-		"README.md",
-		"package.json"
-	],
-	"dependencies": {
-		"@modelcontextprotocol/sdk": "^1.21.0",
-		"zod": "^4.3.6"
-	},
-	"peerDependencies": {
-		"@elizaos/computeruse": "2.0.0-alpha.3",
-		"@elizaos/core": "2.0.0-alpha.3"
-	},
-	"peerDependenciesMeta": {
-		"@elizaos/computeruse": {
-			"optional": true
-		}
-	},
-	"devDependencies": {
-		"@biomejs/biome": "^2.3.11",
-		"@elizaos/plugin-eliza-classic": "2.0.0-alpha.3",
-		"@elizaos/plugin-inmemorydb": "2.0.0-alpha.3",
-		"@elizaos/plugin-mcp": "2.0.0-alpha.3",
-		"typescript": "^5.9.3",
-		"vitest": "^4.0.17"
-	},
-	"scripts": {
-		"dev": "bun --hot build.ts",
-		"test": "vitest run --passWithNoTests",
-		"lint": "bunx @biomejs/biome check --write --unsafe .",
-		"typecheck": "tsc --noEmit",
-		"clean": "rm -rf dist .turbo",
-		"lint:check": "bunx @biomejs/biome check .",
-		"build": "bun run build.ts",
-		"build:ts": "bun run build.ts",
-		"format": "bunx @biomejs/biome format --write .",
-		"format:check": "bunx @biomejs/biome format ."
-	},
-	"agentConfig": {
-		"pluginType": "elizaos:plugin:1.0.0",
-		"pluginParameters": {
-			"COMPUTERUSE_ENABLED": {
-				"type": "boolean",
-				"description": "Enable ComputerUse actions/providers (default: false)",
-				"required": false
-			},
-			"COMPUTERUSE_MODE": {
-				"type": "string",
-				"description": "Execution mode: auto | local | mcp (default: auto)",
-				"required": false
-			},
-			"COMPUTERUSE_MCP_SERVER": {
-				"type": "string",
-				"description": "MCP server name to use for remote control (default: computeruse)",
-				"required": false
-			}
-		}
-	},
-	"publishConfig": {
-		"access": "public"
-	},
-	"milady": {
-		"platforms": [
-			"node"
-		],
-		"runtime": "node",
-		"platformDetails": {
-			"node": "Node.js only (platform: node)"
-		}
-	}
+  "name": "@elizaos/plugin-computeruse",
+  "version": "2.0.11-beta.7",
+  "description": "Desktop automation plugin — screenshots, mouse/keyboard control, browser CDP, and window management for elizaOS agents. Ported from open-computer-use (Apache 2.0).",
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "eliza-source": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts",
+        "default": "./src/index.ts"
+      },
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./*.css": "./dist/*.css",
+    "./*": {
+      "types": "./dist/*.d.ts",
+      "eliza-source": {
+        "types": "./src/*.ts",
+        "import": "./src/*.ts",
+        "default": "./src/*.ts"
+      },
+      "import": "./dist/*.js",
+      "default": "./dist/*.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "package.json"
+  ],
+  "scripts": {
+    "postinstall": "node scripts/ensure-platform-deps.mjs",
+    "build": "bun run build.ts",
+    "test": "vitest run --config vitest.config.ts",
+    "typecheck": "tsgo --noEmit",
+    "validate:ios-device-evidence": "node scripts/validate-ios-device-evidence.mjs"
+  },
+  "peerDependencies": {
+    "@elizaos/core": "2.0.11-beta.7"
+  },
+  "devDependencies": {
+    "@elizaos/core": "2.0.11-beta.7",
+    "@types/node": "^25.2.3",
+    "bun-types": "^1.3.14",
+    "typescript": "^6.0.3",
+    "vitest": "4.1.9"
+  },
+  "dependencies": {
+    "@nut-tree-fork/nut-js": "^4.2.6",
+    "puppeteer-core": "^24.9.0"
+  },
+  "license": "Apache-2.0",
+  "agentConfig": {
+    "pluginType": "elizaos:plugin:1.0.0",
+    "pluginParameters": {
+      "COMPUTER_USE_ENABLED": {
+        "type": "boolean",
+        "description": "Enable computer use capabilities (desktop automation, screenshots, browser control)",
+        "required": false,
+        "default": false,
+        "sensitive": false
+      },
+      "COMPUTER_USE_SCREENSHOT_AFTER_ACTION": {
+        "type": "boolean",
+        "description": "Automatically capture a screenshot after each desktop action",
+        "required": false,
+        "default": true,
+        "sensitive": false
+      },
+      "COMPUTER_USE_ACTION_TIMEOUT_MS": {
+        "type": "number",
+        "description": "Maximum action execution timeout in milliseconds",
+        "required": false,
+        "default": 10000,
+        "sensitive": false
+      },
+      "COMPUTER_USE_APPROVAL_MODE": {
+        "type": "string",
+        "description": "Approval mode for computer-use commands: full_control auto-approves everything, smart_approve auto-approves safe read-only commands, approve_all requires approval for every command, off denies all commands.",
+        "required": false,
+        "default": "smart_approve",
+        "enum": [
+          "full_control",
+          "smart_approve",
+          "approve_all",
+          "off"
+        ],
+        "sensitive": false
+      },
+      "COMPUTER_USE_BROWSER_HEADLESS": {
+        "type": "boolean",
+        "description": "Run browser automation headlessly. Useful for CI and test environments without a visible desktop session.",
+        "required": false,
+        "default": false,
+        "sensitive": false
+      },
+      "ELIZA_COMPUTERUSE_DRIVER": {
+        "type": "string",
+        "description": "Cross-platform input driver: 'nutjs' (default; uses @nut-tree-fork/nut-js native bindings, works on macOS/Linux/Windows) or 'legacy' (per-OS shell tools: cliclick/xdotool/PowerShell).",
+        "required": false,
+        "default": "nutjs",
+        "enum": [
+          "nutjs",
+          "legacy"
+        ],
+        "sensitive": false
+      },
+      "BROWSER_EXECUTE_DISABLED": {
+        "type": "boolean",
+        "description": "browser_execute / executeBrowser always fail closed (GHSA-rcvr-766c-4phv). Use DOM, clickables, click, type, navigate, and screenshot instead.",
+        "required": false,
+        "default": true,
+        "sensitive": false
+      }
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "cdbc876f793d96073d7eb0d09715a031ce0cd32e"
 }

package/dist/actions/click-coordinates.d.ts

@@ -1,3 +0,0 @@
-import type { Action } from "@elizaos/core";
-export declare const computeruseClickCoordinatesAction: Action;
-//# sourceMappingURL=click-coordinates.d.ts.map

package/dist/actions/click-coordinates.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"click-coordinates.d.ts","sourceRoot":"","sources":["../../src/actions/click-coordinates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,MAAM,EAQN,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,iCAAiC,EAAE,MAsL/C,CAAC"}

package/dist/actions/click-coordinates.js

@@ -1,139 +0,0 @@
-import { logger } from "@elizaos/core";
-import { getNumberParam, getStringParam } from "../utils/params.js";
-export const computeruseClickCoordinatesAction = {
-    name: "COMPUTERUSE_CLICK_COORDINATES",
-    description: "Clicks at specific screen coordinates (x, y). Use for precise clicking at known positions.",
-    similes: ["CLICK_AT", "CLICK_POSITION", "CLICK_XY"],
-    parameters: [
-        {
-            name: "x",
-            description: "X coordinate to click at",
-            required: true,
-            schema: { type: "number" },
-            examples: [500, 960],
-        },
-        {
-            name: "y",
-            description: "Y coordinate to click at",
-            required: true,
-            schema: { type: "number" },
-            examples: [300, 540],
-        },
-        {
-            name: "button",
-            description: "Mouse button: 'left', 'right', or 'middle'. Default: 'left'",
-            required: false,
-            schema: { type: "string" },
-            examples: ["left", "right"],
-        },
-        {
-            name: "numClicks",
-            description: "Number of clicks (1=single, 2=double). Default: 1",
-            required: false,
-            schema: { type: "number", default: 1, minimum: 1, maximum: 3 },
-            examples: [1, 2],
-        },
-    ],
-    validate: async (runtime, message, state, options) => {
-        const __avTextRaw = typeof message?.content?.text === "string" ? message.content.text : "";
-        const __avText = __avTextRaw.toLowerCase();
-        const __avKeywords = ["computeruse", "click", "coordinates"];
-        const __avKeywordOk = __avKeywords.length > 0 &&
-            __avKeywords.some((word) => word.length > 0 && __avText.includes(word));
-        const __avRegex = /\b(?:computeruse|click|coordinates)\b/i;
-        const __avRegexOk = __avRegex.test(__avText);
-        const __avSource = String(message?.content?.source ?? message?.source ?? "");
-        const __avExpectedSource = "";
-        const __avSourceOk = __avExpectedSource
-            ? __avSource === __avExpectedSource
-            : Boolean(__avSource ||
-                state ||
-                runtime?.agentId ||
-                runtime?.getService ||
-                runtime?.getSetting);
-        const __avOptions = options && typeof options === "object" ? options : {};
-        const __avInputOk = __avText.trim().length > 0 ||
-            Object.keys(__avOptions).length > 0 ||
-            Boolean(message?.content && typeof message.content === "object");
-        if (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {
-            return false;
-        }
-        const __avLegacyValidate = async (runtime, message, state, options) => {
-            const __avTextRaw = typeof message?.content?.text === "string" ? message.content.text : "";
-            const __avText = __avTextRaw.toLowerCase();
-            const __avKeywords = ["computeruse", "click", "coordinates"];
-            const __avKeywordOk = __avKeywords.length > 0 &&
-                __avKeywords.some((kw) => kw.length > 0 && __avText.includes(kw));
-            const __avRegex = /\b(?:computeruse|click|coordinates)\b/i;
-            const __avRegexOk = __avRegex.test(__avText);
-            const __avSource = String(message?.content?.source ?? message?.source ?? "");
-            const __avExpectedSource = "";
-            const __avSourceOk = __avExpectedSource
-                ? __avSource === __avExpectedSource
-                : Boolean(__avSource || state || runtime?.agentId || runtime?.getService);
-            const __avOptions = options && typeof options === "object" ? options : {};
-            const __avInputOk = __avText.trim().length > 0 ||
-                Object.keys(__avOptions).length > 0 ||
-                Boolean(message?.content && typeof message.content === "object");
-            if (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {
-                return false;
-            }
-            const __avLegacyValidate = async (runtime, _message) => {
-                const service = runtime.getService("computeruse");
-                return !!service && service.isEnabled();
-            };
-            try {
-                return Boolean(await __avLegacyValidate(runtime, message, state, options));
-            }
-            catch {
-                return false;
-            }
-        };
-        try {
-            return Boolean(await __avLegacyValidate(runtime, message, state, options));
-        }
-        catch {
-            return false;
-        }
-    },
-    handler: async (runtime, message, _state, options, callback) => {
-        const service = runtime.getService("computeruse");
-        if (!service) {
-            return { success: false, text: "ComputerUse service not available" };
-        }
-        const params = options?.parameters;
-        const x = getNumberParam(params, "x");
-        const y = getNumberParam(params, "y");
-        const button = getStringParam(params, "button") ?? "left";
-        const numClicks = getNumberParam(params, "numClicks") ?? 1;
-        if (x === undefined || y === undefined) {
-            return { success: false, text: "Missing required parameters: x and y" };
-        }
-        const backend = service.getBackendName();
-        try {
-            await service.clickCoordinates(x, y, button, numClicks);
-            const response = {
-                text: `Clicked at coordinates (${x}, ${y}) with ${button} button`,
-                actions: ["COMPUTERUSE_CLICK_COORDINATES"],
-                source: message.content?.source ?? "action",
-            };
-            await callback?.(response);
-            return {
-                success: true,
-                text: response.text ?? "",
-                values: { x, y, button, numClicks },
-                data: { x, y, button, numClicks, backend },
-            };
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            logger.error(`[computeruse] click_coordinates failed: ${msg}`);
-            return {
-                success: false,
-                text: `ComputerUse click at coordinates failed: ${msg}`,
-                values: { x, y, button, numClicks },
-            };
-        }
-    },
-};
-//# sourceMappingURL=click-coordinates.js.map

package/dist/actions/click-coordinates.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"click-coordinates.js","sourceRoot":"","sources":["../../src/actions/click-coordinates.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpE,MAAM,CAAC,MAAM,iCAAiC,GAAW;IACxD,IAAI,EAAE,+BAA+B;IACrC,WAAW,EACV,4FAA4F;IAC7F,OAAO,EAAE,CAAC,UAAU,EAAE,gBAAgB,EAAE,UAAU,CAAC;IACnD,UAAU,EAAE;QACX;YACC,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,0BAA0B;YACvC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB;QACD;YACC,IAAI,EAAE,GAAG;YACT,WAAW,EAAE,0BAA0B;YACvC,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;SACpB;QACD;YACC,IAAI,EAAE,QAAQ;YACd,WAAW,EACV,6DAA6D;YAC9D,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;SAC3B;QACD;YACC,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,mDAAmD;YAChE,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;YAC9D,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;SAChB;KACD;IAED,QAAQ,EAAE,KAAK,EACd,OAAY,EACZ,OAAY,EACZ,KAAW,EACX,OAAa,EACM,EAAE;QACrB,MAAM,WAAW,GAChB,OAAO,OAAO,EAAE,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QAC7D,MAAM,aAAa,GAClB,YAAY,CAAC,MAAM,GAAG,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,wCAAwC,CAAC;QAC3D,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,MAAM,CACxB,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,OAAO,EAAE,MAAM,IAAI,EAAE,CACjD,CAAC;QACF,MAAM,kBAAkB,GAAG,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,kBAAkB;YACtC,CAAC,CAAC,UAAU,KAAK,kBAAkB;YACnC,CAAC,CAAC,OAAO,CACP,UAAU;gBACT,KAAK;gBACL,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,UAAU;gBACnB,OAAO,EAAE,UAAU,CACpB,CAAC;QACJ,MAAM,WAAW,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,WAAW,GAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,WAAsC,CAAC,CAAC,MAAM,GAAG,CAAC;YAC9D,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAElE,IAAI,CAAC,CAAC,aAAa,IAAI,WAAW,IAAI,YAAY,IAAI,WAAW,CAAC,EAAE,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QAED,MAAM,kBAAkB,GAAG,KAAK,EAC/B,OAAY,EACZ,OAAY,EACZ,KAAW,EACX,OAAa,EACM,EAAE;YACrB,MAAM,WAAW,GAChB,OAAO,OAAO,EAAE,OAAO,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;YAC7D,MAAM,aAAa,GAClB,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,wCAAwC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,UAAU,GAAG,MAAM,CACxB,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,OAAO,EAAE,MAAM,IAAI,EAAE,CACjD,CAAC;YACF,MAAM,kBAAkB,GAAG,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,kBAAkB;gBACtC,CAAC,CAAC,UAAU,KAAK,kBAAkB;gBACnC,CAAC,CAAC,OAAO,CACP,UAAU,IAAI,KAAK,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,UAAU,CAC9D,CAAC;YACJ,MAAM,WAAW,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1E,MAAM,WAAW,GAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,WAAsC,CAAC,CAAC,MAAM,GAAG,CAAC;gBAC9D,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;YAElE,IAAI,CAAC,CAAC,aAAa,IAAI,WAAW,IAAI,YAAY,IAAI,WAAW,CAAC,EAAE,CAAC;gBACpE,OAAO,KAAK,CAAC;YACd,CAAC;YAED,MAAM,kBAAkB,GAAG,KAAK,EAC/B,OAAsB,EACtB,QAAgB,EACG,EAAE;gBACrB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;gBACtE,OAAO,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACzC,CAAC,CAAC;YACF,IAAI,CAAC;gBACJ,OAAO,OAAO,CACb,MAAO,kBAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACnE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,KAAK,CAAC;YACd,CAAC;QACF,CAAC,CAAC;QACF,IAAI,CAAC;YACJ,OAAO,OAAO,CACb,MAAO,kBAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACnE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EACb,OAAsB,EACtB,OAAe,EACf,MAAc,EACd,OAAwB,EACxB,QAA0B,EACF,EAAE;QAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAqB,aAAa,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,mCAAmC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,UAAU,CAAC;QACnC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC;QAC1D,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QAE3D,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACxC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,sCAAsC,EAAE,CAAC;QACzE,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;QAEzC,IAAI,CAAC;YACJ,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;YAExD,MAAM,QAAQ,GAAY;gBACzB,IAAI,EAAE,2BAA2B,CAAC,KAAK,CAAC,UAAU,MAAM,SAAS;gBACjE,OAAO,EAAE,CAAC,+BAA+B,CAAC;gBAC1C,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,QAAQ;aAC3C,CAAC;YACF,MAAM,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC3B,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;gBACnC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE;aAC1C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;YAC/D,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,4CAA4C,GAAG,EAAE;gBACvD,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;aACnC,CAAC;QACH,CAAC;IACF,CAAC;CACD,CAAC","sourcesContent":["import type {\n\tAction,\n\tActionResult,\n\tContent,\n\tHandlerCallback,\n\tHandlerOptions,\n\tIAgentRuntime,\n\tMemory,\n\tState,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport type { ComputerUseService } from \"../services/computeruse-service.js\";\nimport { getNumberParam, getStringParam } from \"../utils/params.js\";\n\nexport const computeruseClickCoordinatesAction: Action = {\n\tname: \"COMPUTERUSE_CLICK_COORDINATES\",\n\tdescription:\n\t\t\"Clicks at specific screen coordinates (x, y). Use for precise clicking at known positions.\",\n\tsimiles: [\"CLICK_AT\", \"CLICK_POSITION\", \"CLICK_XY\"],\n\tparameters: [\n\t\t{\n\t\t\tname: \"x\",\n\t\t\tdescription: \"X coordinate to click at\",\n\t\t\trequired: true,\n\t\t\tschema: { type: \"number\" },\n\t\t\texamples: [500, 960],\n\t\t},\n\t\t{\n\t\t\tname: \"y\",\n\t\t\tdescription: \"Y coordinate to click at\",\n\t\t\trequired: true,\n\t\t\tschema: { type: \"number\" },\n\t\t\texamples: [300, 540],\n\t\t},\n\t\t{\n\t\t\tname: \"button\",\n\t\t\tdescription:\n\t\t\t\t\"Mouse button: 'left', 'right', or 'middle'. Default: 'left'\",\n\t\t\trequired: false,\n\t\t\tschema: { type: \"string\" },\n\t\t\texamples: [\"left\", \"right\"],\n\t\t},\n\t\t{\n\t\t\tname: \"numClicks\",\n\t\t\tdescription: \"Number of clicks (1=single, 2=double). Default: 1\",\n\t\t\trequired: false,\n\t\t\tschema: { type: \"number\", default: 1, minimum: 1, maximum: 3 },\n\t\t\texamples: [1, 2],\n\t\t},\n\t],\n\n\tvalidate: async (\n\t\truntime: any,\n\t\tmessage: any,\n\t\tstate?: any,\n\t\toptions?: any,\n\t): Promise<boolean> => {\n\t\tconst __avTextRaw =\n\t\t\ttypeof message?.content?.text === \"string\" ? message.content.text : \"\";\n\t\tconst __avText = __avTextRaw.toLowerCase();\n\t\tconst __avKeywords = [\"computeruse\", \"click\", \"coordinates\"];\n\t\tconst __avKeywordOk =\n\t\t\t__avKeywords.length > 0 &&\n\t\t\t__avKeywords.some((word) => word.length > 0 && __avText.includes(word));\n\t\tconst __avRegex = /\\b(?:computeruse|click|coordinates)\\b/i;\n\t\tconst __avRegexOk = __avRegex.test(__avText);\n\t\tconst __avSource = String(\n\t\t\tmessage?.content?.source ?? message?.source ?? \"\",\n\t\t);\n\t\tconst __avExpectedSource = \"\";\n\t\tconst __avSourceOk = __avExpectedSource\n\t\t\t? __avSource === __avExpectedSource\n\t\t\t: Boolean(\n\t\t\t\t\t__avSource ||\n\t\t\t\t\t\tstate ||\n\t\t\t\t\t\truntime?.agentId ||\n\t\t\t\t\t\truntime?.getService ||\n\t\t\t\t\t\truntime?.getSetting,\n\t\t\t\t);\n\t\tconst __avOptions = options && typeof options === \"object\" ? options : {};\n\t\tconst __avInputOk =\n\t\t\t__avText.trim().length > 0 ||\n\t\t\tObject.keys(__avOptions as Record<string, unknown>).length > 0 ||\n\t\t\tBoolean(message?.content && typeof message.content === \"object\");\n\n\t\tif (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {\n\t\t\treturn false;\n\t\t}\n\n\t\tconst __avLegacyValidate = async (\n\t\t\truntime: any,\n\t\t\tmessage: any,\n\t\t\tstate?: any,\n\t\t\toptions?: any,\n\t\t): Promise<boolean> => {\n\t\t\tconst __avTextRaw =\n\t\t\t\ttypeof message?.content?.text === \"string\" ? message.content.text : \"\";\n\t\t\tconst __avText = __avTextRaw.toLowerCase();\n\t\t\tconst __avKeywords = [\"computeruse\", \"click\", \"coordinates\"];\n\t\t\tconst __avKeywordOk =\n\t\t\t\t__avKeywords.length > 0 &&\n\t\t\t\t__avKeywords.some((kw) => kw.length > 0 && __avText.includes(kw));\n\t\t\tconst __avRegex = /\\b(?:computeruse|click|coordinates)\\b/i;\n\t\t\tconst __avRegexOk = __avRegex.test(__avText);\n\t\t\tconst __avSource = String(\n\t\t\t\tmessage?.content?.source ?? message?.source ?? \"\",\n\t\t\t);\n\t\t\tconst __avExpectedSource = \"\";\n\t\t\tconst __avSourceOk = __avExpectedSource\n\t\t\t\t? __avSource === __avExpectedSource\n\t\t\t\t: Boolean(\n\t\t\t\t\t\t__avSource || state || runtime?.agentId || runtime?.getService,\n\t\t\t\t\t);\n\t\t\tconst __avOptions = options && typeof options === \"object\" ? options : {};\n\t\t\tconst __avInputOk =\n\t\t\t\t__avText.trim().length > 0 ||\n\t\t\t\tObject.keys(__avOptions as Record<string, unknown>).length > 0 ||\n\t\t\t\tBoolean(message?.content && typeof message.content === \"object\");\n\n\t\t\tif (!(__avKeywordOk && __avRegexOk && __avSourceOk && __avInputOk)) {\n\t\t\t\treturn false;\n\t\t\t}\n\n\t\t\tconst __avLegacyValidate = async (\n\t\t\t\truntime: IAgentRuntime,\n\t\t\t\t_message: Memory,\n\t\t\t): Promise<boolean> => {\n\t\t\t\tconst service = runtime.getService<ComputerUseService>(\"computeruse\");\n\t\t\t\treturn !!service && service.isEnabled();\n\t\t\t};\n\t\t\ttry {\n\t\t\t\treturn Boolean(\n\t\t\t\t\tawait (__avLegacyValidate as any)(runtime, message, state, options),\n\t\t\t\t);\n\t\t\t} catch {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t};\n\t\ttry {\n\t\t\treturn Boolean(\n\t\t\t\tawait (__avLegacyValidate as any)(runtime, message, state, options),\n\t\t\t);\n\t\t} catch {\n\t\t\treturn false;\n\t\t}\n\t},\n\n\thandler: async (\n\t\truntime: IAgentRuntime,\n\t\tmessage: Memory,\n\t\t_state?: State,\n\t\toptions?: HandlerOptions,\n\t\tcallback?: HandlerCallback,\n\t): Promise<ActionResult> => {\n\t\tconst service = runtime.getService<ComputerUseService>(\"computeruse\");\n\t\tif (!service) {\n\t\t\treturn { success: false, text: \"ComputerUse service not available\" };\n\t\t}\n\n\t\tconst params = options?.parameters;\n\t\tconst x = getNumberParam(params, \"x\");\n\t\tconst y = getNumberParam(params, \"y\");\n\t\tconst button = getStringParam(params, \"button\") ?? \"left\";\n\t\tconst numClicks = getNumberParam(params, \"numClicks\") ?? 1;\n\n\t\tif (x === undefined || y === undefined) {\n\t\t\treturn { success: false, text: \"Missing required parameters: x and y\" };\n\t\t}\n\n\t\tconst backend = service.getBackendName();\n\n\t\ttry {\n\t\t\tawait service.clickCoordinates(x, y, button, numClicks);\n\n\t\t\tconst response: Content = {\n\t\t\t\ttext: `Clicked at coordinates (${x}, ${y}) with ${button} button`,\n\t\t\t\tactions: [\"COMPUTERUSE_CLICK_COORDINATES\"],\n\t\t\t\tsource: message.content?.source ?? \"action\",\n\t\t\t};\n\t\t\tawait callback?.(response);\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\ttext: response.text ?? \"\",\n\t\t\t\tvalues: { x, y, button, numClicks },\n\t\t\t\tdata: { x, y, button, numClicks, backend },\n\t\t\t};\n\t\t} catch (err) {\n\t\t\tconst msg = err instanceof Error ? err.message : String(err);\n\t\t\tlogger.error(`[computeruse] click_coordinates failed: ${msg}`);\n\t\t\treturn {\n\t\t\t\tsuccess: false,\n\t\t\t\ttext: `ComputerUse click at coordinates failed: ${msg}`,\n\t\t\t\tvalues: { x, y, button, numClicks },\n\t\t\t};\n\t\t}\n\t},\n};\n"]}

package/dist/actions/click.d.ts

@@ -1,3 +0,0 @@
-import type { Action } from "@elizaos/core";
-export declare const computeruseClickAction: Action;
-//# sourceMappingURL=click.d.ts.map

package/dist/actions/click.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../src/actions/click.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,MAAM,EAQN,MAAM,eAAe,CAAC;AAKvB,eAAO,MAAM,sBAAsB,EAAE,MAwLpC,CAAC"}