@elizaos/plugin-computeruse 2.0.0-alpha.8 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/dist/actions/desktop-handlers.d.ts +20 -0
  2. package/dist/actions/desktop-handlers.d.ts.map +1 -0
  3. package/dist/actions/desktop.d.ts +11 -0
  4. package/dist/actions/desktop.d.ts.map +1 -0
  5. package/dist/actions/helpers.d.ts +33 -0
  6. package/dist/actions/helpers.d.ts.map +1 -0
  7. package/dist/actions/use-computer.d.ts +3 -0
  8. package/dist/actions/use-computer.d.ts.map +1 -0
  9. package/dist/approval-manager.d.ts +29 -0
  10. package/dist/approval-manager.d.ts.map +1 -0
  11. package/dist/index.d.ts +30 -5
  12. package/dist/index.d.ts.map +1 -1
  13. package/dist/index.js +7126 -57
  14. package/dist/index.js.map +34 -1
  15. package/dist/osworld/action-converter.d.ts +35 -0
  16. package/dist/osworld/action-converter.d.ts.map +1 -0
  17. package/dist/osworld/adapter.d.ts +78 -0
  18. package/dist/osworld/adapter.d.ts.map +1 -0
  19. package/dist/osworld/types.d.ts +69 -0
  20. package/dist/osworld/types.d.ts.map +1 -0
  21. package/dist/platform/a11y.d.ts +36 -0
  22. package/dist/platform/a11y.d.ts.map +1 -0
  23. package/dist/platform/browser.d.ts +61 -0
  24. package/dist/platform/browser.d.ts.map +1 -0
  25. package/dist/platform/capabilities.d.ts +10 -0
  26. package/dist/platform/capabilities.d.ts.map +1 -0
  27. package/dist/platform/desktop.d.ts +33 -0
  28. package/dist/platform/desktop.d.ts.map +1 -0
  29. package/dist/platform/driver.d.ts +27 -0
  30. package/dist/platform/driver.d.ts.map +1 -0
  31. package/dist/platform/file-ops.d.ts +10 -0
  32. package/dist/platform/file-ops.d.ts.map +1 -0
  33. package/dist/platform/helpers.d.ts +61 -0
  34. package/dist/platform/helpers.d.ts.map +1 -0
  35. package/dist/platform/nut-driver.d.ts +19 -0
  36. package/dist/platform/nut-driver.d.ts.map +1 -0
  37. package/dist/platform/permissions.d.ts +21 -0
  38. package/dist/platform/permissions.d.ts.map +1 -0
  39. package/dist/platform/screenshot.d.ts +16 -0
  40. package/dist/platform/screenshot.d.ts.map +1 -0
  41. package/dist/platform/security.d.ts +12 -0
  42. package/dist/platform/security.d.ts.map +1 -0
  43. package/dist/platform/terminal.d.ts +38 -0
  44. package/dist/platform/terminal.d.ts.map +1 -0
  45. package/dist/platform/windows-list.d.ts +36 -0
  46. package/dist/platform/windows-list.d.ts.map +1 -0
  47. package/dist/providers/computer-state.d.ts +9 -0
  48. package/dist/providers/computer-state.d.ts.map +1 -0
  49. package/dist/register-routes.d.ts +2 -0
  50. package/dist/register-routes.d.ts.map +1 -0
  51. package/dist/register-routes.js +7111 -0
  52. package/dist/register-routes.js.map +34 -0
  53. package/dist/routes/computer-use-compat-routes.d.ts +29 -0
  54. package/dist/routes/computer-use-compat-routes.d.ts.map +1 -0
  55. package/dist/routes/computer-use-routes.d.ts +3 -0
  56. package/dist/routes/computer-use-routes.d.ts.map +1 -0
  57. package/dist/routes/sandbox-routes.d.ts +53 -0
  58. package/dist/routes/sandbox-routes.d.ts.map +1 -0
  59. package/dist/services/computer-use-service.d.ts +66 -0
  60. package/dist/services/computer-use-service.d.ts.map +1 -0
  61. package/dist/services/desktop-control.d.ts +35 -0
  62. package/dist/services/desktop-control.d.ts.map +1 -0
  63. package/dist/types.d.ts +273 -39
  64. package/dist/types.d.ts.map +1 -1
  65. package/package.json +65 -68
  66. package/dist/actions/click-coordinates.d.ts +0 -3
  67. package/dist/actions/click-coordinates.d.ts.map +0 -1
  68. package/dist/actions/click-coordinates.js +0 -81
  69. package/dist/actions/click-coordinates.js.map +0 -1
  70. package/dist/actions/click.d.ts +0 -3
  71. package/dist/actions/click.d.ts.map +0 -1
  72. package/dist/actions/click.js +0 -76
  73. package/dist/actions/click.js.map +0 -1
  74. package/dist/actions/drag.d.ts +0 -3
  75. package/dist/actions/drag.d.ts.map +0 -1
  76. package/dist/actions/drag.js +0 -87
  77. package/dist/actions/drag.js.map +0 -1
  78. package/dist/actions/get-applications.d.ts +0 -3
  79. package/dist/actions/get-applications.d.ts.map +0 -1
  80. package/dist/actions/get-applications.js +0 -42
  81. package/dist/actions/get-applications.js.map +0 -1
  82. package/dist/actions/get-window-tree.d.ts +0 -3
  83. package/dist/actions/get-window-tree.d.ts.map +0 -1
  84. package/dist/actions/get-window-tree.js +0 -73
  85. package/dist/actions/get-window-tree.js.map +0 -1
  86. package/dist/actions/open-application.d.ts +0 -3
  87. package/dist/actions/open-application.d.ts.map +0 -1
  88. package/dist/actions/open-application.js +0 -57
  89. package/dist/actions/open-application.js.map +0 -1
  90. package/dist/actions/press-key.d.ts +0 -3
  91. package/dist/actions/press-key.d.ts.map +0 -1
  92. package/dist/actions/press-key.js +0 -57
  93. package/dist/actions/press-key.js.map +0 -1
  94. package/dist/actions/screenshot.d.ts +0 -3
  95. package/dist/actions/screenshot.d.ts.map +0 -1
  96. package/dist/actions/screenshot.js +0 -41
  97. package/dist/actions/screenshot.js.map +0 -1
  98. package/dist/actions/scroll.d.ts +0 -3
  99. package/dist/actions/scroll.d.ts.map +0 -1
  100. package/dist/actions/scroll.js +0 -79
  101. package/dist/actions/scroll.js.map +0 -1
  102. package/dist/actions/type.d.ts +0 -3
  103. package/dist/actions/type.d.ts.map +0 -1
  104. package/dist/actions/type.js +0 -103
  105. package/dist/actions/type.js.map +0 -1
  106. package/dist/providers/available-apps.d.ts +0 -3
  107. package/dist/providers/available-apps.d.ts.map +0 -1
  108. package/dist/providers/available-apps.js +0 -22
  109. package/dist/providers/available-apps.js.map +0 -1
  110. package/dist/providers/computeruse-state.d.ts +0 -3
  111. package/dist/providers/computeruse-state.d.ts.map +0 -1
  112. package/dist/providers/computeruse-state.js +0 -35
  113. package/dist/providers/computeruse-state.js.map +0 -1
  114. package/dist/service-registry.d.ts +0 -7
  115. package/dist/service-registry.d.ts.map +0 -1
  116. package/dist/service-registry.js +0 -2
  117. package/dist/service-registry.js.map +0 -1
  118. package/dist/services/computeruse-service.d.ts +0 -36
  119. package/dist/services/computeruse-service.d.ts.map +0 -1
  120. package/dist/services/computeruse-service.js +0 -386
  121. package/dist/services/computeruse-service.js.map +0 -1
  122. package/dist/types.js +0 -11
  123. package/dist/types.js.map +0 -1
  124. package/dist/utils/params.d.ts +0 -5
  125. package/dist/utils/params.d.ts.map +0 -1
  126. package/dist/utils/params.js +0 -13
  127. package/dist/utils/params.js.map +0 -1
@@ -1 +0,0 @@
1
- {"version":3,"file":"computeruse-service.d.ts","sourceRoot":"","sources":["../../src/services/computeruse-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAU,OAAO,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EACL,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EAGvB,MAAM,aAAa,CAAC;AAYrB,qBAAa,kBAAmB,SAAQ,OAAO;IAC7C,MAAM,CAAC,WAAW,SAAiB;IACnC,qBAAqB,SACmF;IAExG,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,WAAW,CAAS;IAG5B,OAAO,CAAC,YAAY,CAAuD;gBAE/D,OAAO,CAAC,EAAE,aAAa;WAStB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAKjE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,OAAO,IAAI,iBAAiB,CAAC,kBAAkB,CAAC;IAIhD,cAAc,IAAI,sBAAsB,GAAG,IAAI;IAI/C,gBAAgB,IAAI,MAAM;IAI1B,SAAS,IAAI,OAAO;IAId,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;YAMpB,iBAAiB;YAgDjB,kBAAkB;YAuBlB,gBAAgB;YAahB,iBAAiB;IAa/B,OAAO,CAAC,MAAM;IAQd,OAAO,CAAC,0BAA0B;IA0B5B,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB/C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B3E,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,OAAO,EAC1B,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;IA8BV,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAiClF,gBAAgB,CACpB,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,MAAM,GAAE,MAAe,EACvB,SAAS,GAAE,MAAU,GACpB,OAAO,CAAC,IAAI,CAAC;IAuBV,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBrC,MAAM,CACV,SAAS,GAAE,MAAe,EAC1B,MAAM,GAAE,MAAU,EAClB,CAAC,CAAC,EAAE,MAAM,EACV,CAAC,CAAC,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC;IAiCV,IAAI,CACR,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IA0BV,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IA8B7B,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAiC3C"}
@@ -1,386 +0,0 @@
1
- import { logger, Service } from "@elizaos/core";
2
- import { computerUseConfigSchema, } from "../types.js";
3
- export class ComputerUseService extends Service {
4
- static serviceType = "computeruse";
5
- capabilityDescription = "Enables the agent to control a computer UI locally (when supported) or via a ComputerUse MCP server";
6
- computeruseConfig;
7
- backendName = null;
8
- initialized = false;
9
- // Lazy-loaded to avoid importing native bindings unless actually used.
10
- localDesktop = null;
11
- constructor(runtime) {
12
- super(runtime);
13
- this.computeruseConfig = computerUseConfigSchema.parse({
14
- COMPUTERUSE_ENABLED: String(process.env.COMPUTERUSE_ENABLED ?? "false"),
15
- COMPUTERUSE_MODE: String(process.env.COMPUTERUSE_MODE ?? "auto"),
16
- COMPUTERUSE_MCP_SERVER: String(process.env.COMPUTERUSE_MCP_SERVER ?? "computeruse"),
17
- });
18
- }
19
- static async start(runtime) {
20
- const service = new ComputerUseService(runtime);
21
- return service;
22
- }
23
- async stop() {
24
- this.localDesktop = null;
25
- this.backendName = null;
26
- this.initialized = false;
27
- }
28
- getMode() {
29
- return this.computeruseConfig.COMPUTERUSE_MODE;
30
- }
31
- getBackendName() {
32
- return this.backendName;
33
- }
34
- getMcpServerName() {
35
- return this.computeruseConfig.COMPUTERUSE_MCP_SERVER;
36
- }
37
- isEnabled() {
38
- return this.computeruseConfig.COMPUTERUSE_ENABLED;
39
- }
40
- async ensureReady() {
41
- if (this.initialized)
42
- return;
43
- this.initialized = true;
44
- await this.initializeBackend();
45
- }
46
- async initializeBackend() {
47
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED) {
48
- logger.info("[computeruse] disabled (COMPUTERUSE_ENABLED=false)");
49
- this.backendName = null;
50
- return;
51
- }
52
- // ── Sandbox mode: force MCP to prevent local desktop control ──────
53
- const isSandboxMode = Boolean(this.runtime && this.runtime.sandboxMode);
54
- if (isSandboxMode) {
55
- logger.info("[computeruse] sandbox mode active — forcing MCP backend (local desktop control disabled)");
56
- this.computeruseConfig.COMPUTERUSE_MODE = "mcp";
57
- await this.ensureMcpBackend();
58
- this.backendName = "mcp";
59
- return;
60
- }
61
- // ── End sandbox mode override ─────────────────────────────────────
62
- const mode = this.computeruseConfig.COMPUTERUSE_MODE;
63
- if (mode === "local") {
64
- await this.ensureLocalBackend();
65
- this.backendName = "local";
66
- return;
67
- }
68
- if (mode === "mcp") {
69
- await this.ensureMcpBackend();
70
- this.backendName = "mcp";
71
- return;
72
- }
73
- // auto - try local on all platforms, fall back to MCP
74
- try {
75
- await this.ensureLocalBackend();
76
- this.backendName = "local";
77
- return;
78
- }
79
- catch (err) {
80
- const msg = err instanceof Error ? err.message : String(err);
81
- logger.warn(`[computeruse] local backend unavailable, falling back to mcp: ${msg}`);
82
- }
83
- await this.ensureMcpBackend();
84
- this.backendName = "mcp";
85
- }
86
- async ensureLocalBackend() {
87
- if (this.localDesktop)
88
- return;
89
- // Import only when needed (native optional deps).
90
- try {
91
- const mod = await import("@elizaos/computeruse");
92
- this.localDesktop = new mod.Desktop();
93
- }
94
- catch (err) {
95
- const msg = err instanceof Error ? err.message : String(err);
96
- const platform = process.platform;
97
- if (platform === "darwin") {
98
- throw new Error(`macOS native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-darwin-* is installed.`);
99
- }
100
- else if (platform === "linux") {
101
- throw new Error(`Linux native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-linux-* is installed.`);
102
- }
103
- throw err;
104
- }
105
- }
106
- async ensureMcpBackend() {
107
- const mcp = await this.waitForMcpService();
108
- const serverName = this.computeruseConfig.COMPUTERUSE_MCP_SERVER;
109
- const servers = mcp.getServers();
110
- const exists = servers.some((s) => s.name === serverName);
111
- if (!exists) {
112
- throw new Error(`MCP server "${serverName}" not configured. Add it under runtime/character settings "mcp.servers".`);
113
- }
114
- }
115
- async waitForMcpService() {
116
- // Services are registered during runtime initialization; depending on plugin order,
117
- // "mcp" may not be available during another service's start hook.
118
- for (let attempt = 0; attempt < 20; attempt++) {
119
- const mcp = this.runtime.getService("mcp");
120
- if (mcp)
121
- return mcp;
122
- await new Promise((resolve) => setTimeout(resolve, 50));
123
- }
124
- throw new Error("MCP service not available. Add @elizaos/plugin-mcp and configure a computeruse server.");
125
- }
126
- getMcp() {
127
- const mcp = this.runtime.getService("mcp");
128
- if (!mcp) {
129
- throw new Error("MCP service not available");
130
- }
131
- return mcp;
132
- }
133
- parseProcessScopedSelector(rawSelector, processHint) {
134
- const selector = rawSelector.trim();
135
- const match = selector.match(/^\s*process:([^\s>]+)\s*(?:>>\s*(.*))?$/);
136
- if (match) {
137
- const process = match[1]?.trim();
138
- const inner = (match[2] ?? "").trim();
139
- if (!process) {
140
- throw new Error("Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'");
141
- }
142
- return { process, selector: inner };
143
- }
144
- const process = processHint?.trim();
145
- if (!process) {
146
- throw new Error("Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'");
147
- }
148
- return { process, selector };
149
- }
150
- async openApplication(appName) {
151
- await this.ensureReady();
152
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
153
- throw new Error("ComputerUse is disabled");
154
- if (this.backendName === "local") {
155
- await this.ensureLocalBackend();
156
- this.localDesktop?.openApplication(appName);
157
- return;
158
- }
159
- if (this.backendName === "mcp") {
160
- const mcp = this.getMcp();
161
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "open_application", {
162
- app_name: appName,
163
- verify_element_exists: "",
164
- verify_element_not_exists: "",
165
- include_tree_after_action: false,
166
- });
167
- return;
168
- }
169
- throw new Error("ComputerUse backend not initialized");
170
- }
171
- async click(selector, timeoutMs, process) {
172
- await this.ensureReady();
173
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
174
- throw new Error("ComputerUse is disabled");
175
- if (this.backendName === "local") {
176
- await this.ensureLocalBackend();
177
- const el = await this.localDesktop?.locator(selector).first(timeoutMs);
178
- if (!el)
179
- throw new Error(`Element not found: ${selector}`);
180
- await el.click();
181
- return;
182
- }
183
- if (this.backendName === "mcp") {
184
- const mcp = this.getMcp();
185
- const parsed = this.parseProcessScopedSelector(selector, process);
186
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "click_element", {
187
- process: parsed.process,
188
- selector: parsed.selector,
189
- timeout_ms: timeoutMs,
190
- verify_element_exists: "",
191
- verify_element_not_exists: "",
192
- highlight_before_action: false,
193
- ui_diff_before_after: false,
194
- });
195
- return;
196
- }
197
- throw new Error("ComputerUse backend not initialized");
198
- }
199
- async typeText(selector, text, timeoutMs, clearBeforeTyping, process) {
200
- await this.ensureReady();
201
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
202
- throw new Error("ComputerUse is disabled");
203
- if (this.backendName === "local") {
204
- await this.ensureLocalBackend();
205
- const el = await this.localDesktop?.locator(selector).first(timeoutMs);
206
- if (!el)
207
- throw new Error(`Element not found: ${selector}`);
208
- el.typeText(text, { clearBeforeTyping });
209
- return;
210
- }
211
- if (this.backendName === "mcp") {
212
- const mcp = this.getMcp();
213
- const parsed = this.parseProcessScopedSelector(selector, process);
214
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "type_into_element", {
215
- process: parsed.process,
216
- selector: parsed.selector,
217
- text_to_type: text,
218
- timeout_ms: timeoutMs,
219
- clear_before_typing: clearBeforeTyping,
220
- highlight_before_action: false,
221
- ui_diff_before_after: false,
222
- });
223
- return;
224
- }
225
- throw new Error("ComputerUse backend not initialized");
226
- }
227
- async getWindowTree(process, title, maxDepth) {
228
- await this.ensureReady();
229
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
230
- throw new Error("ComputerUse is disabled");
231
- if (this.backendName === "local") {
232
- await this.ensureLocalBackend();
233
- const tree = this.localDesktop?.getWindowTree(process, title, undefined);
234
- return JSON.stringify(tree ?? null, null, 2);
235
- }
236
- if (this.backendName === "mcp") {
237
- const mcp = this.getMcp();
238
- const toolArgs = {
239
- process,
240
- include_tree_after_action: true,
241
- };
242
- if (title !== undefined)
243
- toolArgs.title = title;
244
- if (maxDepth !== undefined)
245
- toolArgs.tree_max_depth = maxDepth;
246
- const res = await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "get_window_tree", toolArgs);
247
- const texts = res.content
248
- .filter((c) => c.type === "text")
249
- .map((c) => c.text)
250
- .filter((t) => typeof t === "string");
251
- return texts.join("\n");
252
- }
253
- throw new Error("ComputerUse backend not initialized");
254
- }
255
- async clickCoordinates(x, y, button = "left", numClicks = 1) {
256
- await this.ensureReady();
257
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
258
- throw new Error("ComputerUse is disabled");
259
- if (this.backendName === "local") {
260
- await this.ensureLocalBackend();
261
- this.localDesktop?.clickAtCoordinates(x, y);
262
- return;
263
- }
264
- if (this.backendName === "mcp") {
265
- const mcp = this.getMcp();
266
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "click_at_coordinates", { x, y, button, num_clicks: numClicks });
267
- return;
268
- }
269
- throw new Error("ComputerUse backend not initialized");
270
- }
271
- async pressKey(keys) {
272
- await this.ensureReady();
273
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
274
- throw new Error("ComputerUse is disabled");
275
- if (this.backendName === "local") {
276
- await this.ensureLocalBackend();
277
- this.localDesktop?.pressKey(keys);
278
- return;
279
- }
280
- if (this.backendName === "mcp") {
281
- const mcp = this.getMcp();
282
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "press_key", { key: keys });
283
- return;
284
- }
285
- throw new Error("ComputerUse backend not initialized");
286
- }
287
- async scroll(direction = "down", amount = 3, x, y) {
288
- await this.ensureReady();
289
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
290
- throw new Error("ComputerUse is disabled");
291
- if (this.backendName === "local") {
292
- await this.ensureLocalBackend();
293
- // Local backend: click at position first if specified, then scroll via key presses
294
- if (x !== undefined && y !== undefined) {
295
- this.localDesktop?.clickAtCoordinates(x, y);
296
- }
297
- const scrollKey = direction === "up" ? "scrollup" : "scrolldown";
298
- for (let i = 0; i < amount; i++) {
299
- this.localDesktop?.pressKey(scrollKey);
300
- }
301
- return;
302
- }
303
- if (this.backendName === "mcp") {
304
- const mcp = this.getMcp();
305
- const toolArgs = { direction, amount };
306
- if (x !== undefined)
307
- toolArgs.x = x;
308
- if (y !== undefined)
309
- toolArgs.y = y;
310
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "scroll", toolArgs);
311
- return;
312
- }
313
- throw new Error("ComputerUse backend not initialized");
314
- }
315
- async drag(startX, startY, endX, endY) {
316
- await this.ensureReady();
317
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
318
- throw new Error("ComputerUse is disabled");
319
- if (this.backendName === "local") {
320
- await this.ensureLocalBackend();
321
- // The Desktop native API does not expose a drag method.
322
- // Local backend can only click the start position; full drag requires MCP.
323
- logger.warn("[computeruse] Local drag is limited to click-at-start. Use MCP for full drag.");
324
- this.localDesktop?.clickAtCoordinates(startX, startY);
325
- return;
326
- }
327
- if (this.backendName === "mcp") {
328
- const mcp = this.getMcp();
329
- await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "drag", { start_x: startX, start_y: startY, end_x: endX, end_y: endY });
330
- return;
331
- }
332
- throw new Error("ComputerUse backend not initialized");
333
- }
334
- async screenshot() {
335
- await this.ensureReady();
336
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
337
- throw new Error("ComputerUse is disabled");
338
- if (this.backendName === "local") {
339
- await this.ensureLocalBackend();
340
- const result = this.localDesktop?.captureMonitor(0);
341
- if (result && typeof result === "object" && "toBase64" in result) {
342
- return result.toBase64();
343
- }
344
- return String(result ?? "");
345
- }
346
- if (this.backendName === "mcp") {
347
- const mcp = this.getMcp();
348
- const res = await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "capture_screenshot", {});
349
- const texts = res.content
350
- .filter((c) => c.type === "text")
351
- .map((c) => c.text)
352
- .filter((t) => typeof t === "string");
353
- return texts.join("");
354
- }
355
- throw new Error("ComputerUse backend not initialized");
356
- }
357
- async getApplications() {
358
- await this.ensureReady();
359
- if (!this.computeruseConfig.COMPUTERUSE_ENABLED)
360
- throw new Error("ComputerUse is disabled");
361
- if (this.backendName === "local") {
362
- await this.ensureLocalBackend();
363
- const apps = this.localDesktop?.applications() ?? [];
364
- // Normalize into human-readable names.
365
- const names = [];
366
- for (const app of apps) {
367
- const n = app.name();
368
- if (typeof n === "string" && n.trim().length > 0)
369
- names.push(n);
370
- }
371
- return names;
372
- }
373
- if (this.backendName === "mcp") {
374
- const mcp = this.getMcp();
375
- const res = await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, "get_applications_and_windows_list", {});
376
- // Return text-only summary for now; structured parsing is done in actions/providers if needed.
377
- const texts = res.content
378
- .filter((c) => c.type === "text")
379
- .map((c) => c.text)
380
- .filter((t) => typeof t === "string");
381
- return texts.length > 0 ? texts : ["(see MCP tool output)"];
382
- }
383
- throw new Error("ComputerUse backend not initialized");
384
- }
385
- }
386
- //# sourceMappingURL=computeruse-service.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"computeruse-service.js","sourceRoot":"","sources":["../../src/services/computeruse-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAGL,uBAAuB,GAExB,MAAM,aAAa,CAAC;AAYrB,MAAM,OAAO,kBAAmB,SAAQ,OAAO;IAC7C,MAAM,CAAC,WAAW,GAAG,aAAa,CAAC;IACnC,qBAAqB,GACnB,qGAAqG,CAAC;IAEhG,iBAAiB,CAAoB;IACrC,WAAW,GAAkC,IAAI,CAAC;IAClD,WAAW,GAAG,KAAK,CAAC;IAE5B,uEAAuE;IAC/D,YAAY,GAAkD,IAAI,CAAC;IAE3E,YAAY,OAAuB;QACjC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,KAAK,CAAC;YACrD,mBAAmB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC;YACvE,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC;YAChE,sBAAsB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,aAAa,CAAC;SACpF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAsB;QACvC,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;IACjD,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC;IACvD,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,aAAa,GAAG,OAAO,CAC3B,IAAI,CAAC,OAAO,IAAK,IAAI,CAAC,OAAmC,CAAC,WAAW,CACtE,CAAC;QACF,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CACT,0FAA0F,CAC3F,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChD,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,OAAO;QACT,CAAC;QACD,qEAAqE;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;QACrD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,OAAO;QACT,CAAC;QAED,sDAAsD;QACtD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,iEAAiE,GAAG,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAE9B,kDAAkD;QAClD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YACjD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QACxC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YAClC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,sEAAsE,CAClH,CAAC;YACJ,CAAC;iBAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,qEAAqE,CACjH,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,eAAe,UAAU,0EAA0E,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,oFAAoF;QACpF,kEAAkE;QAClE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAiB,KAAK,CAAC,CAAC;YAC3D,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;YACpB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;IACJ,CAAC;IAEO,MAAM;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAiB,KAAK,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,0BAA0B,CAChC,WAAmB,EACnB,WAAoB;QAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxE,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,kBAAkB,EAAE;gBACpF,QAAQ,EAAE,OAAO;gBACjB,qBAAqB,EAAE,EAAE;gBACzB,yBAAyB,EAAE,EAAE;gBAC7B,yBAAyB,EAAE,KAAK;aACjC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,SAAiB,EAAE,OAAgB;QAC/D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC3D,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClE,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,eAAe,EAAE;gBACjF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE,SAAS;gBACrB,qBAAqB,EAAE,EAAE;gBACzB,yBAAyB,EAAE,EAAE;gBAC7B,uBAAuB,EAAE,KAAK;gBAC9B,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,QAAgB,EAChB,IAAY,EACZ,SAAiB,EACjB,iBAA0B,EAC1B,OAAgB;QAEhB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC3D,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACzC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClE,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,mBAAmB,EAAE;gBACrF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,YAAY,EAAE,IAAI;gBAClB,UAAU,EAAE,SAAS;gBACrB,mBAAmB,EAAE,iBAAiB;gBACtC,uBAAuB,EAAE,KAAK;gBAC9B,oBAAoB,EAAE,KAAK;aAC5B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,KAAc,EAAE,QAAiB;QACpE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAe;gBAC3B,OAAO;gBACP,yBAAyB,EAAE,IAAI;aAChC,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;YAChD,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC;YAC/D,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,iBAAiB,EACjB,QAAQ,CACT,CAAC;YACF,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,CAAS,EACT,CAAS,EACT,SAAiB,MAAM,EACvB,YAAoB,CAAC;QAErB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,sBAAsB,EACtB,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CACxC,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAClC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,WAAW,EACX,EAAE,GAAG,EAAE,IAAI,EAAE,CACd,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,MAAM,CACV,YAAoB,MAAM,EAC1B,SAAiB,CAAC,EAClB,CAAU,EACV,CAAU;QAEV,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,mFAAmF;YACnF,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gBACvC,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9C,CAAC;YACD,MAAM,SAAS,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;YACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzC,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAe,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACnD,IAAI,CAAC,KAAK,SAAS;gBAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,SAAS;gBAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,QAAQ,EACR,QAAQ,CACT,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAc,EACd,MAAc,EACd,IAAY,EACZ,IAAY;QAEZ,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,wDAAwD;YACxD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;YAC7F,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,CAAC,QAAQ,CAChB,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,MAAM,EACN,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAC/D,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;YACpD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,UAAU,IAAI,MAAM,EAAE,CAAC;gBACjE,OAAQ,MAAiC,CAAC,QAAQ,EAAE,CAAC;YACvD,CAAC;YACD,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,oBAAoB,EACpB,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACrD,uCAAuC;YACvC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;gBACrB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAC7C,mCAAmC,EACnC,EAAE,CACH,CAAC;YACF,+FAA+F;YAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC","sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { logger, Service } from \"@elizaos/core\";\nimport type { CallToolResult } from \"@modelcontextprotocol/sdk/types.js\";\nimport {\n type ComputerUseBackendName,\n type ComputerUseConfig,\n computerUseConfigSchema,\n type JsonObject,\n} from \"../types.js\";\n\n// Runtime service shape (from @elizaos/plugin-mcp), referenced structurally to avoid a hard dep.\ninterface McpServiceLike extends Service {\n callTool(\n serverName: string,\n toolName: string,\n toolArguments?: Readonly<JsonObject>\n ): Promise<CallToolResult>;\n getServers(): ReadonlyArray<{ name: string; status: string }>;\n}\n\nexport class ComputerUseService extends Service {\n static serviceType = \"computeruse\";\n capabilityDescription =\n \"Enables the agent to control a computer UI locally (when supported) or via a ComputerUse MCP server\";\n\n private computeruseConfig: ComputerUseConfig;\n private backendName: ComputerUseBackendName | null = null;\n private initialized = false;\n\n // Lazy-loaded to avoid importing native bindings unless actually used.\n private localDesktop: import(\"@elizaos/computeruse\").Desktop | null = null;\n\n constructor(runtime?: IAgentRuntime) {\n super(runtime);\n this.computeruseConfig = computerUseConfigSchema.parse({\n COMPUTERUSE_ENABLED: String(process.env.COMPUTERUSE_ENABLED ?? \"false\"),\n COMPUTERUSE_MODE: String(process.env.COMPUTERUSE_MODE ?? \"auto\"),\n COMPUTERUSE_MCP_SERVER: String(process.env.COMPUTERUSE_MCP_SERVER ?? \"computeruse\"),\n });\n }\n\n static async start(runtime: IAgentRuntime): Promise<ComputerUseService> {\n const service = new ComputerUseService(runtime);\n return service;\n }\n\n async stop(): Promise<void> {\n this.localDesktop = null;\n this.backendName = null;\n this.initialized = false;\n }\n\n getMode(): ComputerUseConfig[\"COMPUTERUSE_MODE\"] {\n return this.computeruseConfig.COMPUTERUSE_MODE;\n }\n\n getBackendName(): ComputerUseBackendName | null {\n return this.backendName;\n }\n\n getMcpServerName(): string {\n return this.computeruseConfig.COMPUTERUSE_MCP_SERVER;\n }\n\n isEnabled(): boolean {\n return this.computeruseConfig.COMPUTERUSE_ENABLED;\n }\n\n async ensureReady(): Promise<void> {\n if (this.initialized) return;\n this.initialized = true;\n await this.initializeBackend();\n }\n\n private async initializeBackend(): Promise<void> {\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) {\n logger.info(\"[computeruse] disabled (COMPUTERUSE_ENABLED=false)\");\n this.backendName = null;\n return;\n }\n\n // ── Sandbox mode: force MCP to prevent local desktop control ──────\n const isSandboxMode = Boolean(\n this.runtime && (this.runtime as Record<string, unknown>).sandboxMode,\n );\n if (isSandboxMode) {\n logger.info(\n \"[computeruse] sandbox mode active — forcing MCP backend (local desktop control disabled)\",\n );\n this.computeruseConfig.COMPUTERUSE_MODE = \"mcp\";\n await this.ensureMcpBackend();\n this.backendName = \"mcp\";\n return;\n }\n // ── End sandbox mode override ─────────────────────────────────────\n\n const mode = this.computeruseConfig.COMPUTERUSE_MODE;\n if (mode === \"local\") {\n await this.ensureLocalBackend();\n this.backendName = \"local\";\n return;\n }\n if (mode === \"mcp\") {\n await this.ensureMcpBackend();\n this.backendName = \"mcp\";\n return;\n }\n\n // auto - try local on all platforms, fall back to MCP\n try {\n await this.ensureLocalBackend();\n this.backendName = \"local\";\n return;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n logger.warn(`[computeruse] local backend unavailable, falling back to mcp: ${msg}`);\n }\n\n await this.ensureMcpBackend();\n this.backendName = \"mcp\";\n }\n\n private async ensureLocalBackend(): Promise<void> {\n if (this.localDesktop) return;\n\n // Import only when needed (native optional deps).\n try {\n const mod = await import(\"@elizaos/computeruse\");\n this.localDesktop = new mod.Desktop();\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n const platform = process.platform;\n if (platform === \"darwin\") {\n throw new Error(\n `macOS native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-darwin-* is installed.`\n );\n } else if (platform === \"linux\") {\n throw new Error(\n `Linux native bindings not available: ${msg}. Use MCP mode or ensure @elizaos/computeruse-linux-* is installed.`\n );\n }\n throw err;\n }\n }\n\n private async ensureMcpBackend(): Promise<void> {\n const mcp = await this.waitForMcpService();\n\n const serverName = this.computeruseConfig.COMPUTERUSE_MCP_SERVER;\n const servers = mcp.getServers();\n const exists = servers.some((s) => s.name === serverName);\n if (!exists) {\n throw new Error(\n `MCP server \"${serverName}\" not configured. Add it under runtime/character settings \"mcp.servers\".`\n );\n }\n }\n\n private async waitForMcpService(): Promise<McpServiceLike> {\n // Services are registered during runtime initialization; depending on plugin order,\n // \"mcp\" may not be available during another service's start hook.\n for (let attempt = 0; attempt < 20; attempt++) {\n const mcp = this.runtime.getService<McpServiceLike>(\"mcp\");\n if (mcp) return mcp;\n await new Promise<void>((resolve) => setTimeout(resolve, 50));\n }\n throw new Error(\n \"MCP service not available. Add @elizaos/plugin-mcp and configure a computeruse server.\"\n );\n }\n\n private getMcp(): McpServiceLike {\n const mcp = this.runtime.getService<McpServiceLike>(\"mcp\");\n if (!mcp) {\n throw new Error(\"MCP service not available\");\n }\n return mcp;\n }\n\n private parseProcessScopedSelector(\n rawSelector: string,\n processHint?: string\n ): { process: string; selector: string } {\n const selector = rawSelector.trim();\n const match = selector.match(/^\\s*process:([^\\s>]+)\\s*(?:>>\\s*(.*))?$/);\n if (match) {\n const process = match[1]?.trim();\n const inner = (match[2] ?? \"\").trim();\n if (!process) {\n throw new Error(\n \"Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'\"\n );\n }\n return { process, selector: inner };\n }\n\n const process = processHint?.trim();\n if (!process) {\n throw new Error(\n \"Missing process. Provide parameters.process or prefix selector with 'process:<name> >> ...'\"\n );\n }\n return { process, selector };\n }\n\n async openApplication(appName: string): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n this.localDesktop?.openApplication(appName);\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"open_application\", {\n app_name: appName,\n verify_element_exists: \"\",\n verify_element_not_exists: \"\",\n include_tree_after_action: false,\n });\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async click(selector: string, timeoutMs: number, process?: string): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n const el = await this.localDesktop?.locator(selector).first(timeoutMs);\n if (!el) throw new Error(`Element not found: ${selector}`);\n await el.click();\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n const parsed = this.parseProcessScopedSelector(selector, process);\n await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"click_element\", {\n process: parsed.process,\n selector: parsed.selector,\n timeout_ms: timeoutMs,\n verify_element_exists: \"\",\n verify_element_not_exists: \"\",\n highlight_before_action: false,\n ui_diff_before_after: false,\n });\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async typeText(\n selector: string,\n text: string,\n timeoutMs: number,\n clearBeforeTyping: boolean,\n process?: string\n ): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n const el = await this.localDesktop?.locator(selector).first(timeoutMs);\n if (!el) throw new Error(`Element not found: ${selector}`);\n el.typeText(text, { clearBeforeTyping });\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n const parsed = this.parseProcessScopedSelector(selector, process);\n await mcp.callTool(this.computeruseConfig.COMPUTERUSE_MCP_SERVER, \"type_into_element\", {\n process: parsed.process,\n selector: parsed.selector,\n text_to_type: text,\n timeout_ms: timeoutMs,\n clear_before_typing: clearBeforeTyping,\n highlight_before_action: false,\n ui_diff_before_after: false,\n });\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async getWindowTree(process: string, title?: string, maxDepth?: number): Promise<string> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n const tree = this.localDesktop?.getWindowTree(process, title, undefined);\n return JSON.stringify(tree ?? null, null, 2);\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n const toolArgs: JsonObject = {\n process,\n include_tree_after_action: true,\n };\n if (title !== undefined) toolArgs.title = title;\n if (maxDepth !== undefined) toolArgs.tree_max_depth = maxDepth;\n const res = await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"get_window_tree\",\n toolArgs\n );\n const texts = res.content\n .filter((c) => c.type === \"text\")\n .map((c) => c.text)\n .filter((t) => typeof t === \"string\");\n return texts.join(\"\\n\");\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async clickCoordinates(\n x: number,\n y: number,\n button: string = \"left\",\n numClicks: number = 1\n ): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n this.localDesktop?.clickAtCoordinates(x, y);\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"click_at_coordinates\",\n { x, y, button, num_clicks: numClicks }\n );\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async pressKey(keys: string): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n this.localDesktop?.pressKey(keys);\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"press_key\",\n { key: keys }\n );\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async scroll(\n direction: string = \"down\",\n amount: number = 3,\n x?: number,\n y?: number\n ): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n // Local backend: click at position first if specified, then scroll via key presses\n if (x !== undefined && y !== undefined) {\n this.localDesktop?.clickAtCoordinates(x, y);\n }\n const scrollKey = direction === \"up\" ? \"scrollup\" : \"scrolldown\";\n for (let i = 0; i < amount; i++) {\n this.localDesktop?.pressKey(scrollKey);\n }\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n const toolArgs: JsonObject = { direction, amount };\n if (x !== undefined) toolArgs.x = x;\n if (y !== undefined) toolArgs.y = y;\n await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"scroll\",\n toolArgs\n );\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async drag(\n startX: number,\n startY: number,\n endX: number,\n endY: number\n ): Promise<void> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n // The Desktop native API does not expose a drag method.\n // Local backend can only click the start position; full drag requires MCP.\n logger.warn(\"[computeruse] Local drag is limited to click-at-start. Use MCP for full drag.\");\n this.localDesktop?.clickAtCoordinates(startX, startY);\n return;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"drag\",\n { start_x: startX, start_y: startY, end_x: endX, end_y: endY }\n );\n return;\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async screenshot(): Promise<string> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n const result = this.localDesktop?.captureMonitor(0);\n if (result && typeof result === \"object\" && \"toBase64\" in result) {\n return (result as { toBase64(): string }).toBase64();\n }\n return String(result ?? \"\");\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n const res = await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"capture_screenshot\",\n {}\n );\n const texts = res.content\n .filter((c) => c.type === \"text\")\n .map((c) => c.text)\n .filter((t) => typeof t === \"string\");\n return texts.join(\"\");\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n\n async getApplications(): Promise<string[]> {\n await this.ensureReady();\n if (!this.computeruseConfig.COMPUTERUSE_ENABLED) throw new Error(\"ComputerUse is disabled\");\n\n if (this.backendName === \"local\") {\n await this.ensureLocalBackend();\n const apps = this.localDesktop?.applications() ?? [];\n // Normalize into human-readable names.\n const names: string[] = [];\n for (const app of apps) {\n const n = app.name();\n if (typeof n === \"string\" && n.trim().length > 0) names.push(n);\n }\n return names;\n }\n\n if (this.backendName === \"mcp\") {\n const mcp = this.getMcp();\n const res = await mcp.callTool(\n this.computeruseConfig.COMPUTERUSE_MCP_SERVER,\n \"get_applications_and_windows_list\",\n {}\n );\n // Return text-only summary for now; structured parsing is done in actions/providers if needed.\n const texts = res.content\n .filter((c) => c.type === \"text\")\n .map((c) => c.text)\n .filter((t) => typeof t === \"string\");\n return texts.length > 0 ? texts : [\"(see MCP tool output)\"];\n }\n\n throw new Error(\"ComputerUse backend not initialized\");\n }\n}\n"]}
package/dist/types.js DELETED
@@ -1,11 +0,0 @@
1
- import { z } from "zod";
2
- export const computerUseConfigSchema = z.object({
3
- COMPUTERUSE_ENABLED: z
4
- .string()
5
- .optional()
6
- .default("false")
7
- .transform((val) => val === "true"),
8
- COMPUTERUSE_MODE: z.enum(["auto", "local", "mcp"]).optional().default("auto"),
9
- COMPUTERUSE_MCP_SERVER: z.string().optional().default("computeruse"),
10
- });
11
- //# sourceMappingURL=types.js.map
package/dist/types.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,mBAAmB,EAAE,CAAC;SACnB,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,OAAO,CAAC,OAAO,CAAC;SAChB,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC;IACrC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7E,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC;CACrE,CAAC,CAAC","sourcesContent":["import { z } from \"zod\";\n\nexport type JsonPrimitive = string | number | boolean | null;\nexport type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue };\nexport type JsonObject = { [key: string]: JsonValue };\n\nexport type ComputerUseMode = \"auto\" | \"local\" | \"mcp\";\n\nexport const computerUseConfigSchema = z.object({\n COMPUTERUSE_ENABLED: z\n .string()\n .optional()\n .default(\"false\")\n .transform((val) => val === \"true\"),\n COMPUTERUSE_MODE: z.enum([\"auto\", \"local\", \"mcp\"]).optional().default(\"auto\"),\n COMPUTERUSE_MCP_SERVER: z.string().optional().default(\"computeruse\"),\n});\n\nexport type ComputerUseConfig = z.infer<typeof computerUseConfigSchema>;\n\nexport type ComputerUseBackendName = \"local\" | \"mcp\";\n\nexport interface ComputerUseOpenApplicationInput {\n readonly name: string;\n}\n\nexport interface ComputerUseClickInput {\n readonly process?: string;\n readonly selector: string;\n readonly timeoutMs: number;\n}\n\nexport interface ComputerUseTypeInput {\n readonly process?: string;\n readonly selector: string;\n readonly text: string;\n readonly timeoutMs: number;\n readonly clearBeforeTyping: boolean;\n}\n\nexport interface ComputerUseGetWindowTreeInput {\n readonly process: string;\n readonly title?: string;\n readonly maxDepth?: number;\n}\n"]}
@@ -1,5 +0,0 @@
1
- import type { ActionParameters } from "@elizaos/core";
2
- export declare function getStringParam(params: ActionParameters | undefined, name: string): string | undefined;
3
- export declare function getNumberParam(params: ActionParameters | undefined, name: string): number | undefined;
4
- export declare function getBooleanParam(params: ActionParameters | undefined, name: string): boolean | undefined;
5
- //# sourceMappingURL=params.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/utils/params.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,wBAAgB,cAAc,CAC5B,MAAM,EAAE,gBAAgB,GAAG,SAAS,EACpC,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,SAAS,CAGpB;AAED,wBAAgB,cAAc,CAC5B,MAAM,EAAE,gBAAgB,GAAG,SAAS,EACpC,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,SAAS,CAGpB;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,gBAAgB,GAAG,SAAS,EACpC,IAAI,EAAE,MAAM,GACX,OAAO,GAAG,SAAS,CAGrB"}
@@ -1,13 +0,0 @@
1
- export function getStringParam(params, name) {
2
- const val = params?.[name];
3
- return typeof val === "string" ? val : undefined;
4
- }
5
- export function getNumberParam(params, name) {
6
- const val = params?.[name];
7
- return typeof val === "number" ? val : undefined;
8
- }
9
- export function getBooleanParam(params, name) {
10
- const val = params?.[name];
11
- return typeof val === "boolean" ? val : undefined;
12
- }
13
- //# sourceMappingURL=params.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"params.js","sourceRoot":"","sources":["../../src/utils/params.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,cAAc,CAC5B,MAAoC,EACpC,IAAY;IAEZ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3B,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,MAAoC,EACpC,IAAY;IAEZ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3B,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,MAAoC,EACpC,IAAY;IAEZ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3B,OAAO,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;AACpD,CAAC","sourcesContent":["import type { ActionParameters } from \"@elizaos/core\";\n\nexport function getStringParam(\n params: ActionParameters | undefined,\n name: string\n): string | undefined {\n const val = params?.[name];\n return typeof val === \"string\" ? val : undefined;\n}\n\nexport function getNumberParam(\n params: ActionParameters | undefined,\n name: string\n): number | undefined {\n const val = params?.[name];\n return typeof val === \"number\" ? val : undefined;\n}\n\nexport function getBooleanParam(\n params: ActionParameters | undefined,\n name: string\n): boolean | undefined {\n const val = params?.[name];\n return typeof val === \"boolean\" ? val : undefined;\n}\n"]}