@midscene/computer 1.9.8 → 1.10.1-beta-20260624112700.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +83 -11
- package/dist/es/index.mjs +83 -11
- package/dist/lib/cli.js +83 -11
- package/dist/lib/index.js +83 -11
- package/dist/types/index.d.ts +19 -5
- package/package.json +3 -8
- package/dist/es/mcp-server.mjs +0 -2088
- package/dist/lib/mcp-server.js +0 -2141
- package/dist/types/mcp-server.d.ts +0 -245
package/dist/es/mcp-server.mjs
DELETED
|
@@ -1,2088 +0,0 @@
|
|
|
1
|
-
import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
|
|
2
|
-
import { Agent } from "@midscene/core/agent";
|
|
3
|
-
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
4
|
-
import { chmodSync, existsSync, statSync } from "node:fs";
|
|
5
|
-
import { createRequire } from "node:module";
|
|
6
|
-
import { dirname, resolve as external_node_path_resolve } from "node:path";
|
|
7
|
-
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { defineAction, defineActionsFromInputPrimitives } from "@midscene/core/device";
|
|
9
|
-
import { sleep } from "@midscene/core/utils";
|
|
10
|
-
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
11
|
-
import { getDebug } from "@midscene/shared/logger";
|
|
12
|
-
import screenshot_desktop from "screenshot-desktop";
|
|
13
|
-
import node_assert from "node:assert";
|
|
14
|
-
import { once } from "node:events";
|
|
15
|
-
import { createInterface } from "node:readline";
|
|
16
|
-
import { z } from "@midscene/core";
|
|
17
|
-
import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/mcp/agent-behavior-init-args";
|
|
18
|
-
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
19
|
-
function _define_property(obj, key, value) {
|
|
20
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
21
|
-
value: value,
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true
|
|
25
|
-
});
|
|
26
|
-
else obj[key] = value;
|
|
27
|
-
return obj;
|
|
28
|
-
}
|
|
29
|
-
class ComputerInputDriver {
|
|
30
|
-
destroy() {
|
|
31
|
-
if (this.destroyed) return;
|
|
32
|
-
this.destroyed = true;
|
|
33
|
-
this.rejectPendingInputDelays();
|
|
34
|
-
}
|
|
35
|
-
getScreenSize() {
|
|
36
|
-
return this.getLibnutOrThrow('getScreenSize').getScreenSize();
|
|
37
|
-
}
|
|
38
|
-
getMousePos() {
|
|
39
|
-
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
40
|
-
}
|
|
41
|
-
moveMouse(x, y) {
|
|
42
|
-
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
43
|
-
}
|
|
44
|
-
focusActiveWindow() {
|
|
45
|
-
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
46
|
-
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
47
|
-
try {
|
|
48
|
-
const handle = lib.getActiveWindow();
|
|
49
|
-
if (!handle) return false;
|
|
50
|
-
lib.focusWindow(handle);
|
|
51
|
-
return true;
|
|
52
|
-
} catch (error) {
|
|
53
|
-
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
54
|
-
return false;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
getActiveWindowRect() {
|
|
58
|
-
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
59
|
-
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
60
|
-
try {
|
|
61
|
-
const handle = lib.getActiveWindow();
|
|
62
|
-
if (!handle) return null;
|
|
63
|
-
const rect = lib.getWindowRect(handle);
|
|
64
|
-
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
65
|
-
return rect;
|
|
66
|
-
} catch (error) {
|
|
67
|
-
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
mouseClick(button, double) {
|
|
72
|
-
const lib = this.getLibnutOrThrow('mouseClick');
|
|
73
|
-
if (void 0 !== double) lib.mouseClick(button, double);
|
|
74
|
-
else if (void 0 !== button) lib.mouseClick(button);
|
|
75
|
-
else lib.mouseClick();
|
|
76
|
-
}
|
|
77
|
-
mouseToggle(state, button = 'left') {
|
|
78
|
-
this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
|
|
79
|
-
}
|
|
80
|
-
scrollMouse(x, y) {
|
|
81
|
-
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
82
|
-
}
|
|
83
|
-
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
84
|
-
this.assertActive('emitScrollDetents');
|
|
85
|
-
for(let i = 0; i < detents; i++){
|
|
86
|
-
this.scrollMouse(dx, dy);
|
|
87
|
-
if (i < detents - 1) await this.delay(delayMs);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
keyTap(key, modifiers) {
|
|
91
|
-
const lib = this.getLibnutOrThrow('keyTap');
|
|
92
|
-
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
93
|
-
else lib.keyTap(key);
|
|
94
|
-
}
|
|
95
|
-
sendKeyViaAppleScript(key, modifiers = []) {
|
|
96
|
-
this.assertActive('sendKeyViaAppleScript');
|
|
97
|
-
this.options.sendKeyViaAppleScript(key, modifiers);
|
|
98
|
-
}
|
|
99
|
-
sendKey(key, modifiers = []) {
|
|
100
|
-
if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
|
|
101
|
-
if (modifiers.length > 0) this.keyTap(key, modifiers);
|
|
102
|
-
else this.keyTap(key);
|
|
103
|
-
}
|
|
104
|
-
runPhasedScroll(direction, pixels, steps) {
|
|
105
|
-
this.assertActive('runPhasedScroll');
|
|
106
|
-
return this.options.runPhasedScroll(direction, pixels, steps);
|
|
107
|
-
}
|
|
108
|
-
async delay(ms) {
|
|
109
|
-
this.assertActive('delay');
|
|
110
|
-
return new Promise((resolve, reject)=>{
|
|
111
|
-
const waitRef = {
|
|
112
|
-
timeoutId: setTimeout(()=>{
|
|
113
|
-
this.pendingInputDelayWaits.delete(waitRef);
|
|
114
|
-
try {
|
|
115
|
-
this.assertActive('delay');
|
|
116
|
-
resolve();
|
|
117
|
-
} catch (error) {
|
|
118
|
-
reject(error);
|
|
119
|
-
}
|
|
120
|
-
}, ms),
|
|
121
|
-
reject
|
|
122
|
-
};
|
|
123
|
-
this.pendingInputDelayWaits.add(waitRef);
|
|
124
|
-
});
|
|
125
|
-
}
|
|
126
|
-
async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
127
|
-
const currentPos = this.getMousePos();
|
|
128
|
-
for(let i = 1; i <= steps; i++){
|
|
129
|
-
const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
|
|
130
|
-
const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
|
|
131
|
-
this.moveMouse(stepX, stepY);
|
|
132
|
-
await this.delay(stepDelay);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
async withMouseButton(button, run) {
|
|
136
|
-
this.mouseToggle('down', button);
|
|
137
|
-
try {
|
|
138
|
-
return await run();
|
|
139
|
-
} finally{
|
|
140
|
-
this.releaseMouseButton(button);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
getLibnutOrThrow(methodName) {
|
|
144
|
-
this.assertActive(methodName);
|
|
145
|
-
const libnut = this.options.getLibnut();
|
|
146
|
-
node_assert(libnut, 'libnut not initialized');
|
|
147
|
-
return libnut;
|
|
148
|
-
}
|
|
149
|
-
assertActive(methodName) {
|
|
150
|
-
if (this.destroyed) throw this.createDestroyedError(methodName);
|
|
151
|
-
}
|
|
152
|
-
createDestroyedError(methodName) {
|
|
153
|
-
return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
|
|
154
|
-
}
|
|
155
|
-
releaseMouseButton(button) {
|
|
156
|
-
try {
|
|
157
|
-
const libnut = this.options.getLibnut();
|
|
158
|
-
node_assert(libnut, 'libnut not initialized');
|
|
159
|
-
libnut.mouseToggle('up', button);
|
|
160
|
-
} catch (error) {
|
|
161
|
-
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
rejectPendingInputDelays() {
|
|
165
|
-
const error = this.createDestroyedError('in-flight input');
|
|
166
|
-
for (const waitRef of this.pendingInputDelayWaits){
|
|
167
|
-
clearTimeout(waitRef.timeoutId);
|
|
168
|
-
waitRef.reject(error);
|
|
169
|
-
}
|
|
170
|
-
this.pendingInputDelayWaits.clear();
|
|
171
|
-
}
|
|
172
|
-
constructor(options){
|
|
173
|
-
_define_property(this, "options", void 0);
|
|
174
|
-
_define_property(this, "destroyed", void 0);
|
|
175
|
-
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
176
|
-
this.options = options;
|
|
177
|
-
this.destroyed = false;
|
|
178
|
-
this.pendingInputDelayWaits = new Set();
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
const debugXvfb = getDebug('computer:xvfb');
|
|
182
|
-
function checkXvfbInstalled() {
|
|
183
|
-
try {
|
|
184
|
-
execSync('which Xvfb', {
|
|
185
|
-
stdio: 'ignore'
|
|
186
|
-
});
|
|
187
|
-
return true;
|
|
188
|
-
} catch {
|
|
189
|
-
return false;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
function findAvailableDisplay(startFrom = 99) {
|
|
193
|
-
for(let n = startFrom; n < startFrom + 100; n++)if (!existsSync(`/tmp/.X${n}-lock`)) return n;
|
|
194
|
-
throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
|
|
195
|
-
}
|
|
196
|
-
function startXvfb(options) {
|
|
197
|
-
const resolution = options?.resolution || '1920x1080x24';
|
|
198
|
-
const displayNum = options?.displayNumber ?? findAvailableDisplay();
|
|
199
|
-
const display = `:${displayNum}`;
|
|
200
|
-
return new Promise((resolve, reject)=>{
|
|
201
|
-
debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
|
|
202
|
-
const xvfbProcess = spawn('Xvfb', [
|
|
203
|
-
display,
|
|
204
|
-
'-screen',
|
|
205
|
-
'0',
|
|
206
|
-
resolution,
|
|
207
|
-
'-ac',
|
|
208
|
-
'-nolisten',
|
|
209
|
-
'tcp'
|
|
210
|
-
], {
|
|
211
|
-
stdio: 'ignore'
|
|
212
|
-
});
|
|
213
|
-
let settled = false;
|
|
214
|
-
xvfbProcess.on('error', (err)=>{
|
|
215
|
-
if (!settled) {
|
|
216
|
-
settled = true;
|
|
217
|
-
reject(new Error(`Failed to start Xvfb: ${err.message}`));
|
|
218
|
-
}
|
|
219
|
-
});
|
|
220
|
-
xvfbProcess.on('exit', (code)=>{
|
|
221
|
-
if (!settled) {
|
|
222
|
-
settled = true;
|
|
223
|
-
reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
|
|
224
|
-
}
|
|
225
|
-
});
|
|
226
|
-
const instance = {
|
|
227
|
-
process: xvfbProcess,
|
|
228
|
-
display,
|
|
229
|
-
stop () {
|
|
230
|
-
try {
|
|
231
|
-
xvfbProcess.kill('SIGTERM');
|
|
232
|
-
} catch {}
|
|
233
|
-
}
|
|
234
|
-
};
|
|
235
|
-
setTimeout(()=>{
|
|
236
|
-
if (!settled) {
|
|
237
|
-
settled = true;
|
|
238
|
-
debugXvfb(`Xvfb started on display ${display}`);
|
|
239
|
-
resolve(instance);
|
|
240
|
-
}
|
|
241
|
-
}, 500);
|
|
242
|
-
});
|
|
243
|
-
}
|
|
244
|
-
function needsXvfb(explicitOpt) {
|
|
245
|
-
if ('linux' !== process.platform) return false;
|
|
246
|
-
return true === explicitOpt;
|
|
247
|
-
}
|
|
248
|
-
function device_define_property(obj, key, value) {
|
|
249
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
250
|
-
value: value,
|
|
251
|
-
enumerable: true,
|
|
252
|
-
configurable: true,
|
|
253
|
-
writable: true
|
|
254
|
-
});
|
|
255
|
-
else obj[key] = value;
|
|
256
|
-
return obj;
|
|
257
|
-
}
|
|
258
|
-
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
259
|
-
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
260
|
-
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
261
|
-
const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
262
|
-
const MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
263
|
-
const CLICK_SETTLE_DELAY = 50;
|
|
264
|
-
const CLICK_HOLD_DURATION = 100;
|
|
265
|
-
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
266
|
-
const INPUT_FOCUS_DELAY = 300;
|
|
267
|
-
const INPUT_CLEAR_DELAY = 150;
|
|
268
|
-
const SCROLL_STEP_DELAY = 100;
|
|
269
|
-
const SCROLL_COMPLETE_DELAY = 500;
|
|
270
|
-
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
271
|
-
const EDGE_SCROLL_STEPS = 400;
|
|
272
|
-
const PHASED_PIXELS_PER_STEP = 30;
|
|
273
|
-
const PHASED_MIN_STEPS = 10;
|
|
274
|
-
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
275
|
-
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
276
|
-
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
277
|
-
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
278
|
-
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
279
|
-
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
280
|
-
const EDGE_SCROLL_SPEC = {
|
|
281
|
-
scrollToTop: {
|
|
282
|
-
direction: 'up',
|
|
283
|
-
key: 'home',
|
|
284
|
-
libnut: [
|
|
285
|
-
0,
|
|
286
|
-
1
|
|
287
|
-
]
|
|
288
|
-
},
|
|
289
|
-
scrollToBottom: {
|
|
290
|
-
direction: 'down',
|
|
291
|
-
key: 'end',
|
|
292
|
-
libnut: [
|
|
293
|
-
0,
|
|
294
|
-
-1
|
|
295
|
-
]
|
|
296
|
-
},
|
|
297
|
-
scrollToLeft: {
|
|
298
|
-
direction: 'left',
|
|
299
|
-
key: 'home',
|
|
300
|
-
libnut: [
|
|
301
|
-
-1,
|
|
302
|
-
0
|
|
303
|
-
]
|
|
304
|
-
},
|
|
305
|
-
scrollToRight: {
|
|
306
|
-
direction: 'right',
|
|
307
|
-
key: 'end',
|
|
308
|
-
libnut: [
|
|
309
|
-
1,
|
|
310
|
-
0
|
|
311
|
-
]
|
|
312
|
-
}
|
|
313
|
-
};
|
|
314
|
-
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
315
|
-
return: 36,
|
|
316
|
-
enter: 36,
|
|
317
|
-
tab: 48,
|
|
318
|
-
space: 49,
|
|
319
|
-
backspace: 51,
|
|
320
|
-
delete: 51,
|
|
321
|
-
escape: 53,
|
|
322
|
-
forwarddelete: 117,
|
|
323
|
-
left: 123,
|
|
324
|
-
right: 124,
|
|
325
|
-
down: 125,
|
|
326
|
-
up: 126,
|
|
327
|
-
home: 115,
|
|
328
|
-
end: 119,
|
|
329
|
-
pageup: 116,
|
|
330
|
-
pagedown: 121,
|
|
331
|
-
f1: 122,
|
|
332
|
-
f2: 120,
|
|
333
|
-
f3: 99,
|
|
334
|
-
f4: 118,
|
|
335
|
-
f5: 96,
|
|
336
|
-
f6: 97,
|
|
337
|
-
f7: 98,
|
|
338
|
-
f8: 100,
|
|
339
|
-
f9: 101,
|
|
340
|
-
f10: 109,
|
|
341
|
-
f11: 103,
|
|
342
|
-
f12: 111
|
|
343
|
-
};
|
|
344
|
-
const APPLESCRIPT_MODIFIER_MAP = {
|
|
345
|
-
command: 'command down',
|
|
346
|
-
cmd: 'command down',
|
|
347
|
-
control: 'control down',
|
|
348
|
-
ctrl: 'control down',
|
|
349
|
-
shift: 'shift down',
|
|
350
|
-
alt: 'option down',
|
|
351
|
-
option: 'option down',
|
|
352
|
-
meta: 'command down'
|
|
353
|
-
};
|
|
354
|
-
function sendKeyViaAppleScript(key, modifiers = []) {
|
|
355
|
-
const lowerKey = key.toLowerCase();
|
|
356
|
-
const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
|
|
357
|
-
const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
|
|
358
|
-
const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
|
|
359
|
-
let script;
|
|
360
|
-
if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
|
|
361
|
-
else {
|
|
362
|
-
const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
363
|
-
script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
|
|
364
|
-
}
|
|
365
|
-
debugDevice('sendKeyViaAppleScript', {
|
|
366
|
-
key,
|
|
367
|
-
modifiers,
|
|
368
|
-
script
|
|
369
|
-
});
|
|
370
|
-
execFileSync("osascript", [
|
|
371
|
-
'-e',
|
|
372
|
-
script
|
|
373
|
-
]);
|
|
374
|
-
}
|
|
375
|
-
let device_libnut = null;
|
|
376
|
-
let libnutLoadError = null;
|
|
377
|
-
async function getLibnut() {
|
|
378
|
-
if (device_libnut) return device_libnut;
|
|
379
|
-
if (libnutLoadError) throw libnutLoadError;
|
|
380
|
-
try {
|
|
381
|
-
const require = createRequire(import.meta.url);
|
|
382
|
-
const libnutModule = require('@computer-use/libnut/dist/import_libnut');
|
|
383
|
-
device_libnut = libnutModule.libnut;
|
|
384
|
-
if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
|
|
385
|
-
return device_libnut;
|
|
386
|
-
} catch (error) {
|
|
387
|
-
libnutLoadError = error;
|
|
388
|
-
throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
const debugDevice = getDebug('computer:device');
|
|
392
|
-
const warnDevice = getDebug('computer:device', {
|
|
393
|
-
console: true
|
|
394
|
-
});
|
|
395
|
-
const debugComputerInput = getDebug('computer:input', {
|
|
396
|
-
console: true
|
|
397
|
-
});
|
|
398
|
-
const WINDOWS_UIPI_DOC_URL = 'https://midscenejs.com/computer-getting-started#windows-clicks-have-no-effect-on-some-apps';
|
|
399
|
-
function resolvePackageRoot(helperName) {
|
|
400
|
-
const require = createRequire(import.meta.url);
|
|
401
|
-
let pkgRoot = null;
|
|
402
|
-
try {
|
|
403
|
-
pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
|
|
404
|
-
} catch {
|
|
405
|
-
const hereDir = dirname(fileURLToPath(import.meta.url));
|
|
406
|
-
for (const candidate of [
|
|
407
|
-
external_node_path_resolve(hereDir, '..'),
|
|
408
|
-
external_node_path_resolve(hereDir, '../..')
|
|
409
|
-
])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
|
|
410
|
-
pkgRoot = candidate;
|
|
411
|
-
break;
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
if (!pkgRoot) {
|
|
415
|
-
debugDevice(`${helperName}: cannot locate @midscene/computer package root`);
|
|
416
|
-
return null;
|
|
417
|
-
}
|
|
418
|
-
return pkgRoot;
|
|
419
|
-
}
|
|
420
|
-
let phasedScrollBinaryPath;
|
|
421
|
-
function getPhasedScrollBinary() {
|
|
422
|
-
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
423
|
-
if ('darwin' !== process.platform) {
|
|
424
|
-
phasedScrollBinaryPath = null;
|
|
425
|
-
return null;
|
|
426
|
-
}
|
|
427
|
-
const pkgRoot = resolvePackageRoot('phased-scroll');
|
|
428
|
-
if (!pkgRoot) {
|
|
429
|
-
phasedScrollBinaryPath = null;
|
|
430
|
-
return null;
|
|
431
|
-
}
|
|
432
|
-
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
|
|
433
|
-
if (!existsSync(binPath)) {
|
|
434
|
-
debugDevice('phased-scroll binary not found at', binPath);
|
|
435
|
-
phasedScrollBinaryPath = null;
|
|
436
|
-
return null;
|
|
437
|
-
}
|
|
438
|
-
try {
|
|
439
|
-
const st = statSync(binPath);
|
|
440
|
-
if ((73 & st.mode) === 0) {
|
|
441
|
-
chmodSync(binPath, 493);
|
|
442
|
-
debugDevice('phased-scroll: restored executable bit on', binPath);
|
|
443
|
-
}
|
|
444
|
-
} catch (err) {
|
|
445
|
-
debugDevice('phased-scroll: chmod self-heal failed', err);
|
|
446
|
-
}
|
|
447
|
-
phasedScrollBinaryPath = binPath;
|
|
448
|
-
return binPath;
|
|
449
|
-
}
|
|
450
|
-
let displayInfoBinaryPath;
|
|
451
|
-
function getDisplayInfoBinary() {
|
|
452
|
-
if (void 0 !== displayInfoBinaryPath) return displayInfoBinaryPath;
|
|
453
|
-
if ('darwin' !== process.platform) {
|
|
454
|
-
displayInfoBinaryPath = null;
|
|
455
|
-
return null;
|
|
456
|
-
}
|
|
457
|
-
const pkgRoot = resolvePackageRoot('display-info');
|
|
458
|
-
if (!pkgRoot) {
|
|
459
|
-
displayInfoBinaryPath = null;
|
|
460
|
-
return null;
|
|
461
|
-
}
|
|
462
|
-
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/display-info');
|
|
463
|
-
if (!existsSync(binPath)) {
|
|
464
|
-
debugDevice('display-info binary not found at', binPath);
|
|
465
|
-
displayInfoBinaryPath = null;
|
|
466
|
-
return null;
|
|
467
|
-
}
|
|
468
|
-
displayInfoBinaryPath = binPath;
|
|
469
|
-
return binPath;
|
|
470
|
-
}
|
|
471
|
-
function isFiniteNumber(value) {
|
|
472
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
473
|
-
}
|
|
474
|
-
function isDarwinDisplayGeometry(value) {
|
|
475
|
-
if (!value || 'object' != typeof value) return false;
|
|
476
|
-
const candidate = value;
|
|
477
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
|
|
478
|
-
}
|
|
479
|
-
function readDarwinDisplayGeometries() {
|
|
480
|
-
const bin = getDisplayInfoBinary();
|
|
481
|
-
if (!bin) return [];
|
|
482
|
-
try {
|
|
483
|
-
const output = execFileSync(bin, [], {
|
|
484
|
-
encoding: 'utf8',
|
|
485
|
-
stdio: [
|
|
486
|
-
'ignore',
|
|
487
|
-
'pipe',
|
|
488
|
-
'pipe'
|
|
489
|
-
]
|
|
490
|
-
});
|
|
491
|
-
const parsed = JSON.parse(output);
|
|
492
|
-
return Array.isArray(parsed.displays) ? parsed.displays.filter(isDarwinDisplayGeometry) : [];
|
|
493
|
-
} catch (error) {
|
|
494
|
-
debugDevice('display-info helper failed:', error);
|
|
495
|
-
return [];
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
function readDarwinFrontmostApplication() {
|
|
499
|
-
try {
|
|
500
|
-
const output = execFileSync("osascript", [
|
|
501
|
-
'-e',
|
|
502
|
-
'tell application "System Events"\nset frontApp to first application process whose frontmost is true\nreturn (unix id of frontApp as string) & " " & (name of frontApp as string)\nend tell'
|
|
503
|
-
]).toString().trim();
|
|
504
|
-
const [pidText, ...nameParts] = output.split('\t');
|
|
505
|
-
const pid = Number(pidText);
|
|
506
|
-
if (!Number.isInteger(pid) || pid <= 0) return;
|
|
507
|
-
return {
|
|
508
|
-
pid,
|
|
509
|
-
name: nameParts.join('\t')
|
|
510
|
-
};
|
|
511
|
-
} catch (error) {
|
|
512
|
-
debugDevice('Failed to read macOS frontmost application:', error);
|
|
513
|
-
return;
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
|
|
517
|
-
await inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
518
|
-
const current = inputDriver.getMousePos();
|
|
519
|
-
debugComputerInput('tap mouse moved %o', {
|
|
520
|
-
reason,
|
|
521
|
-
target: {
|
|
522
|
-
x: targetX,
|
|
523
|
-
y: targetY
|
|
524
|
-
},
|
|
525
|
-
current,
|
|
526
|
-
drift: {
|
|
527
|
-
x: current.x - targetX,
|
|
528
|
-
y: current.y - targetY
|
|
529
|
-
}
|
|
530
|
-
});
|
|
531
|
-
await inputDriver.withMouseButton('left', async ()=>{
|
|
532
|
-
debugComputerInput('tap mouse down %o', {
|
|
533
|
-
reason
|
|
534
|
-
});
|
|
535
|
-
await inputDriver.delay(holdDuration);
|
|
536
|
-
});
|
|
537
|
-
debugComputerInput('tap mouse up %o', {
|
|
538
|
-
reason
|
|
539
|
-
});
|
|
540
|
-
}
|
|
541
|
-
function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
542
|
-
if (!displays.length) return;
|
|
543
|
-
const screenIndex = void 0 === displayId || '' === displayId ? 0 : Number(displayId);
|
|
544
|
-
if (!Number.isInteger(screenIndex) || screenIndex < 0) return void debugDevice('Invalid macOS display id for display geometry:', displayId);
|
|
545
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
546
|
-
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
547
|
-
}
|
|
548
|
-
function resolveDisplayGeometry(displayId) {
|
|
549
|
-
if ('darwin' !== process.platform) return;
|
|
550
|
-
return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
551
|
-
}
|
|
552
|
-
function mapDisplayLocalPointToGlobal(point, geometry) {
|
|
553
|
-
if (!geometry) return point;
|
|
554
|
-
return {
|
|
555
|
-
x: point.x + geometry.bounds.x,
|
|
556
|
-
y: point.y + geometry.bounds.y
|
|
557
|
-
};
|
|
558
|
-
}
|
|
559
|
-
let phasedScrollExecWarned = false;
|
|
560
|
-
function runPhasedScroll(direction, pixels, steps) {
|
|
561
|
-
const bin = getPhasedScrollBinary();
|
|
562
|
-
if (!bin) return false;
|
|
563
|
-
try {
|
|
564
|
-
const res = spawnSync(bin, [
|
|
565
|
-
direction,
|
|
566
|
-
String(Math.max(1, Math.round(pixels))),
|
|
567
|
-
String(steps)
|
|
568
|
-
], {
|
|
569
|
-
stdio: 'ignore'
|
|
570
|
-
});
|
|
571
|
-
if (0 === res.status) return true;
|
|
572
|
-
if (!phasedScrollExecWarned) {
|
|
573
|
-
phasedScrollExecWarned = true;
|
|
574
|
-
const hint = null === res.status ? `signal ${res.signal ?? 'unknown'}; the binary may not be executable (npm tarball extraction can drop the +x bit) or may be blocked by quarantine. Try: chmod +x "${bin}"` : 'this usually means Accessibility permission has not been granted to the host process (System Settings → Privacy & Security → Accessibility)';
|
|
575
|
-
console.warn(`[@midscene/computer] phased-scroll helper failed (exit=${res.status}, signal=${res.signal ?? 'none'}); falling back to keyboard/libnut. ${hint}`);
|
|
576
|
-
}
|
|
577
|
-
debugDevice('phased-scroll exited non-zero', res.status, res.signal, res.error);
|
|
578
|
-
return false;
|
|
579
|
-
} catch (err) {
|
|
580
|
-
if (!phasedScrollExecWarned) {
|
|
581
|
-
phasedScrollExecWarned = true;
|
|
582
|
-
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
583
|
-
}
|
|
584
|
-
debugDevice('phased-scroll spawn failed', err);
|
|
585
|
-
return false;
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
const KEY_NAME_MAP = {
|
|
589
|
-
windows: 'win',
|
|
590
|
-
win: 'win',
|
|
591
|
-
ctrl: 'control',
|
|
592
|
-
esc: 'escape',
|
|
593
|
-
del: 'delete',
|
|
594
|
-
ins: 'insert',
|
|
595
|
-
pgup: 'pageup',
|
|
596
|
-
pgdn: 'pagedown',
|
|
597
|
-
arrowup: 'up',
|
|
598
|
-
arrowdown: 'down',
|
|
599
|
-
arrowleft: 'left',
|
|
600
|
-
arrowright: 'right',
|
|
601
|
-
volumedown: 'audio_vol_down',
|
|
602
|
-
volumeup: 'audio_vol_up',
|
|
603
|
-
mediavolumedown: 'audio_vol_down',
|
|
604
|
-
mediavolumeup: 'audio_vol_up',
|
|
605
|
-
mute: 'audio_mute',
|
|
606
|
-
mediamute: 'audio_mute',
|
|
607
|
-
mediaplay: 'audio_play',
|
|
608
|
-
mediapause: 'audio_pause',
|
|
609
|
-
mediaplaypause: 'audio_play',
|
|
610
|
-
mediastop: 'audio_stop',
|
|
611
|
-
medianexttrack: 'audio_next',
|
|
612
|
-
mediaprevioustrack: 'audio_prev',
|
|
613
|
-
medianext: 'audio_next',
|
|
614
|
-
mediaprev: 'audio_prev'
|
|
615
|
-
};
|
|
616
|
-
const PRIMARY_KEY_MAP = {
|
|
617
|
-
command: 'cmd',
|
|
618
|
-
cmd: 'cmd',
|
|
619
|
-
meta: 'meta',
|
|
620
|
-
control: 'control',
|
|
621
|
-
ctrl: 'control',
|
|
622
|
-
shift: 'shift',
|
|
623
|
-
alt: 'alt',
|
|
624
|
-
option: 'alt'
|
|
625
|
-
};
|
|
626
|
-
function normalizeKeyName(key) {
|
|
627
|
-
const lowerKey = key.toLowerCase();
|
|
628
|
-
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
629
|
-
}
|
|
630
|
-
function normalizePrimaryKey(key) {
|
|
631
|
-
const lowerKey = key.toLowerCase();
|
|
632
|
-
if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
|
|
633
|
-
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
634
|
-
}
|
|
635
|
-
class ComputerDevice {
|
|
636
|
-
describe() {
|
|
637
|
-
return this.description || 'Computer Device';
|
|
638
|
-
}
|
|
639
|
-
static async listDisplays() {
|
|
640
|
-
try {
|
|
641
|
-
const displays = await screenshot_desktop.listDisplays();
|
|
642
|
-
return displays.map((d)=>({
|
|
643
|
-
id: String(d.id),
|
|
644
|
-
name: d.name || `Display ${d.id}`,
|
|
645
|
-
primary: d.primary || false
|
|
646
|
-
}));
|
|
647
|
-
} catch (error) {
|
|
648
|
-
debugDevice(`Failed to list displays: ${error}`);
|
|
649
|
-
return [];
|
|
650
|
-
}
|
|
651
|
-
}
|
|
652
|
-
async connect() {
|
|
653
|
-
debugDevice('Connecting to computer device');
|
|
654
|
-
try {
|
|
655
|
-
const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
|
|
656
|
-
if (needsXvfb(headless)) {
|
|
657
|
-
if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
|
|
658
|
-
this.xvfbInstance = await startXvfb({
|
|
659
|
-
resolution: this.options?.xvfbResolution
|
|
660
|
-
});
|
|
661
|
-
process.env.DISPLAY = this.xvfbInstance.display;
|
|
662
|
-
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
663
|
-
this.xvfbCleanup = ()=>{
|
|
664
|
-
if (this.xvfbInstance) {
|
|
665
|
-
this.xvfbInstance.stop();
|
|
666
|
-
this.xvfbInstance = void 0;
|
|
667
|
-
}
|
|
668
|
-
};
|
|
669
|
-
process.on('exit', this.xvfbCleanup);
|
|
670
|
-
process.on('SIGINT', this.xvfbCleanup);
|
|
671
|
-
process.on('SIGTERM', this.xvfbCleanup);
|
|
672
|
-
}
|
|
673
|
-
device_libnut = await getLibnut();
|
|
674
|
-
this.displayGeometry = resolveDisplayGeometry(this.displayId);
|
|
675
|
-
const size = await this.size();
|
|
676
|
-
const displays = await ComputerDevice.listDisplays();
|
|
677
|
-
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
678
|
-
this.description = `
|
|
679
|
-
Type: Computer
|
|
680
|
-
Platform: ${process.platform}
|
|
681
|
-
Display: ${this.displayId || 'Primary'}
|
|
682
|
-
Screen Size: ${size.width}x${size.height}
|
|
683
|
-
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
684
|
-
`;
|
|
685
|
-
debugDevice('Computer device connected', this.description);
|
|
686
|
-
await this.healthCheck();
|
|
687
|
-
} catch (error) {
|
|
688
|
-
if (this.xvfbInstance) {
|
|
689
|
-
this.xvfbInstance.stop();
|
|
690
|
-
this.xvfbInstance = void 0;
|
|
691
|
-
}
|
|
692
|
-
debugDevice(`Failed to connect: ${error}`);
|
|
693
|
-
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
async healthCheck() {
|
|
697
|
-
console.log('[HealthCheck] Starting health check...');
|
|
698
|
-
console.log("[HealthCheck] @midscene/computer v1.9.8");
|
|
699
|
-
console.log('[HealthCheck] Taking screenshot...');
|
|
700
|
-
const screenshotTimeout = 15000;
|
|
701
|
-
let timeoutId;
|
|
702
|
-
const timeoutPromise = new Promise((_, reject)=>{
|
|
703
|
-
timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
|
|
704
|
-
});
|
|
705
|
-
const base64 = await Promise.race([
|
|
706
|
-
this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
|
|
707
|
-
timeoutPromise
|
|
708
|
-
]);
|
|
709
|
-
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
710
|
-
console.log('[HealthCheck] Moving mouse...');
|
|
711
|
-
const startPos = this.inputDriver.getMousePos();
|
|
712
|
-
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
713
|
-
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
714
|
-
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
715
|
-
const targetX = startPos.x + offsetX;
|
|
716
|
-
const targetY = startPos.y + offsetY;
|
|
717
|
-
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
718
|
-
this.inputDriver.moveMouse(targetX, targetY);
|
|
719
|
-
await sleep(50);
|
|
720
|
-
const movedPos = this.inputDriver.getMousePos();
|
|
721
|
-
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
722
|
-
const deltaX = Math.abs(movedPos.x - targetX);
|
|
723
|
-
const deltaY = Math.abs(movedPos.y - targetY);
|
|
724
|
-
if (deltaX > 5 || deltaY > 5) {
|
|
725
|
-
const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
|
|
726
|
-
warnDevice(msg);
|
|
727
|
-
}
|
|
728
|
-
if ('win32' === process.platform && !this.isRunningAsAdmin()) {
|
|
729
|
-
const hint = [
|
|
730
|
-
'Heads-up: Midscene is not running as Administrator.',
|
|
731
|
-
'If clicks or key presses have no effect while the cursor still moves to the right position,',
|
|
732
|
-
`see the Windows permission troubleshooting guide: ${WINDOWS_UIPI_DOC_URL}`
|
|
733
|
-
].join(' ');
|
|
734
|
-
warnDevice(`[HealthCheck] ${hint}`);
|
|
735
|
-
}
|
|
736
|
-
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
737
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
738
|
-
console.log('[HealthCheck] Listing monitors...');
|
|
739
|
-
const displays = await ComputerDevice.listDisplays();
|
|
740
|
-
if (displays.length > 0) {
|
|
741
|
-
console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
|
|
742
|
-
for (const display of displays){
|
|
743
|
-
const primaryTag = display.primary ? ' (primary)' : '';
|
|
744
|
-
console.log(`[HealthCheck] - id=${display.id}, name=${display.name}${primaryTag}`);
|
|
745
|
-
}
|
|
746
|
-
} else console.log('[HealthCheck] No monitors detected');
|
|
747
|
-
console.log('[HealthCheck] Health check passed');
|
|
748
|
-
}
|
|
749
|
-
isRunningAsAdmin() {
|
|
750
|
-
if ('win32' !== process.platform) return false;
|
|
751
|
-
if (void 0 !== this.adminCheckCache) return this.adminCheckCache;
|
|
752
|
-
try {
|
|
753
|
-
execSync('net session', {
|
|
754
|
-
stdio: 'pipe'
|
|
755
|
-
});
|
|
756
|
-
this.adminCheckCache = true;
|
|
757
|
-
} catch {
|
|
758
|
-
this.adminCheckCache = false;
|
|
759
|
-
}
|
|
760
|
-
return this.adminCheckCache;
|
|
761
|
-
}
|
|
762
|
-
async screenshotBase64() {
|
|
763
|
-
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
764
|
-
debugDevice('Taking screenshot', {
|
|
765
|
-
displayId: this.displayId
|
|
766
|
-
});
|
|
767
|
-
const options = {
|
|
768
|
-
format: 'png'
|
|
769
|
-
};
|
|
770
|
-
if (void 0 !== this.displayId) if ('darwin' === process.platform) {
|
|
771
|
-
const screenIndex = Number(this.displayId);
|
|
772
|
-
if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
|
|
773
|
-
} else options.screen = this.displayId;
|
|
774
|
-
debugDevice('Screenshot options', options);
|
|
775
|
-
const MAX_ATTEMPTS = 3;
|
|
776
|
-
const RETRY_DELAY_MS = 300;
|
|
777
|
-
let lastRawMessage = '';
|
|
778
|
-
for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
|
|
779
|
-
const buffer = await screenshot_desktop(options);
|
|
780
|
-
if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
|
|
781
|
-
return createImgBase64ByFormat('png', buffer.toString('base64'));
|
|
782
|
-
} catch (error) {
|
|
783
|
-
lastRawMessage = error instanceof Error ? error.message : String(error);
|
|
784
|
-
const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
|
|
785
|
-
const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
|
|
786
|
-
debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
|
|
787
|
-
if (!willRetry) break;
|
|
788
|
-
await sleep(RETRY_DELAY_MS);
|
|
789
|
-
}
|
|
790
|
-
if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
|
|
791
|
-
|
|
792
|
-
Please follow these steps:
|
|
793
|
-
1. Open System Settings > Privacy & Security > Screen Recording
|
|
794
|
-
2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
|
|
795
|
-
3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
|
|
796
|
-
|
|
797
|
-
Original error: ${lastRawMessage}`);
|
|
798
|
-
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
799
|
-
}
|
|
800
|
-
async size() {
|
|
801
|
-
if (this.displayGeometry) return {
|
|
802
|
-
width: Math.round(this.displayGeometry.bounds.width),
|
|
803
|
-
height: Math.round(this.displayGeometry.bounds.height)
|
|
804
|
-
};
|
|
805
|
-
try {
|
|
806
|
-
const screenSize = this.inputDriver.getScreenSize();
|
|
807
|
-
return {
|
|
808
|
-
width: screenSize.width,
|
|
809
|
-
height: screenSize.height
|
|
810
|
-
};
|
|
811
|
-
} catch (error) {
|
|
812
|
-
debugDevice(`Failed to get screen size: ${error}`);
|
|
813
|
-
throw new Error(`Failed to get screen size: ${error}`);
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
toGlobalPoint(point) {
|
|
817
|
-
return mapDisplayLocalPointToGlobal(point, this.displayGeometry);
|
|
818
|
-
}
|
|
819
|
-
async typeViaClipboard(text) {
|
|
820
|
-
debugDevice('Using clipboard to input text', {
|
|
821
|
-
textLength: text.length,
|
|
822
|
-
preview: text.substring(0, 20)
|
|
823
|
-
});
|
|
824
|
-
const clipboardy = await import("clipboardy");
|
|
825
|
-
const oldClipboard = await clipboardy.default.read().catch(()=>'');
|
|
826
|
-
try {
|
|
827
|
-
await clipboardy.default.write(text);
|
|
828
|
-
await this.inputDriver.delay(50);
|
|
829
|
-
if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
|
|
830
|
-
'command'
|
|
831
|
-
]);
|
|
832
|
-
else {
|
|
833
|
-
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
834
|
-
this.inputDriver.keyTap('v', [
|
|
835
|
-
modifier
|
|
836
|
-
]);
|
|
837
|
-
}
|
|
838
|
-
await this.inputDriver.delay(100);
|
|
839
|
-
} finally{
|
|
840
|
-
if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
|
|
841
|
-
debugDevice('Failed to restore clipboard content');
|
|
842
|
-
});
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
async smartTypeString(text) {
|
|
846
|
-
await this.typeViaClipboard(text);
|
|
847
|
-
}
|
|
848
|
-
async selectAllAndDelete() {
|
|
849
|
-
if (this.useAppleScript) {
|
|
850
|
-
this.inputDriver.sendKeyViaAppleScript('a', [
|
|
851
|
-
'command'
|
|
852
|
-
]);
|
|
853
|
-
await this.inputDriver.delay(50);
|
|
854
|
-
this.inputDriver.sendKeyViaAppleScript('backspace', []);
|
|
855
|
-
return;
|
|
856
|
-
}
|
|
857
|
-
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
858
|
-
this.inputDriver.keyTap('a', [
|
|
859
|
-
modifier
|
|
860
|
-
]);
|
|
861
|
-
await this.inputDriver.delay(50);
|
|
862
|
-
this.inputDriver.keyTap('backspace');
|
|
863
|
-
}
|
|
864
|
-
async pressKeyboardShortcut(keyName) {
|
|
865
|
-
const keys = keyName.split('+');
|
|
866
|
-
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
867
|
-
const key = normalizePrimaryKey(keys[keys.length - 1]);
|
|
868
|
-
debugDevice('KeyboardPress', {
|
|
869
|
-
original: keyName,
|
|
870
|
-
key,
|
|
871
|
-
modifiers,
|
|
872
|
-
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
873
|
-
});
|
|
874
|
-
this.inputDriver.sendKey(key, modifiers);
|
|
875
|
-
}
|
|
876
|
-
resolveUntargetedScrollPoint(screenSize) {
|
|
877
|
-
if ('win32' === process.platform) {
|
|
878
|
-
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
879
|
-
if (activeWindowRect) return {
|
|
880
|
-
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
881
|
-
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
882
|
-
};
|
|
883
|
-
}
|
|
884
|
-
return this.toGlobalPoint({
|
|
885
|
-
x: screenSize.width / 2,
|
|
886
|
-
y: screenSize.height / 2
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
async moveMouseToScrollTarget(param) {
|
|
890
|
-
if (param.locate) {
|
|
891
|
-
const element = param.locate;
|
|
892
|
-
const [x, y] = element.center;
|
|
893
|
-
const point = this.toGlobalPoint({
|
|
894
|
-
x,
|
|
895
|
-
y
|
|
896
|
-
});
|
|
897
|
-
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
898
|
-
return;
|
|
899
|
-
}
|
|
900
|
-
const screenSize = await this.size();
|
|
901
|
-
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
902
|
-
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
903
|
-
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
904
|
-
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
905
|
-
return screenSize;
|
|
906
|
-
}
|
|
907
|
-
async performScroll(param) {
|
|
908
|
-
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
909
|
-
const scrollType = param?.scrollType;
|
|
910
|
-
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
911
|
-
if (edgeSpec) {
|
|
912
|
-
if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
913
|
-
if (this.useAppleScript) {
|
|
914
|
-
this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
|
|
915
|
-
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
916
|
-
return;
|
|
917
|
-
}
|
|
918
|
-
const [ux, uy] = edgeSpec.libnut;
|
|
919
|
-
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
920
|
-
return;
|
|
921
|
-
}
|
|
922
|
-
if ('singleAction' === scrollType || !scrollType) {
|
|
923
|
-
const direction = param?.direction || 'down';
|
|
924
|
-
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
925
|
-
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
926
|
-
let distance = param?.distance ?? void 0;
|
|
927
|
-
if (!distance) {
|
|
928
|
-
screenSize ??= await this.size();
|
|
929
|
-
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
930
|
-
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
931
|
-
}
|
|
932
|
-
if (isKnownDirection) {
|
|
933
|
-
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
934
|
-
if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
935
|
-
}
|
|
936
|
-
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
937
|
-
if (!screenSize) screenSize = await this.size();
|
|
938
|
-
const pages = Math.max(1, Math.round(distance / screenSize.height));
|
|
939
|
-
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
940
|
-
for(let i = 0; i < pages; i++){
|
|
941
|
-
this.inputDriver.sendKeyViaAppleScript(key);
|
|
942
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
943
|
-
}
|
|
944
|
-
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
945
|
-
return;
|
|
946
|
-
}
|
|
947
|
-
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
948
|
-
const directionUnit = {
|
|
949
|
-
up: [
|
|
950
|
-
0,
|
|
951
|
-
1
|
|
952
|
-
],
|
|
953
|
-
down: [
|
|
954
|
-
0,
|
|
955
|
-
-1
|
|
956
|
-
],
|
|
957
|
-
left: [
|
|
958
|
-
-1,
|
|
959
|
-
0
|
|
960
|
-
],
|
|
961
|
-
right: [
|
|
962
|
-
1,
|
|
963
|
-
0
|
|
964
|
-
]
|
|
965
|
-
};
|
|
966
|
-
const [ux, uy] = directionUnit[direction] || [
|
|
967
|
-
0,
|
|
968
|
-
-1
|
|
969
|
-
];
|
|
970
|
-
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
971
|
-
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
972
|
-
return;
|
|
973
|
-
}
|
|
974
|
-
throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
|
|
975
|
-
}
|
|
976
|
-
actionSpace() {
|
|
977
|
-
const defaultActions = [
|
|
978
|
-
...defineActionsFromInputPrimitives(this.inputPrimitives)
|
|
979
|
-
];
|
|
980
|
-
const platformActions = Object.values(createPlatformActions());
|
|
981
|
-
const customActions = this.options?.customActions || [];
|
|
982
|
-
return [
|
|
983
|
-
...defaultActions,
|
|
984
|
-
...platformActions,
|
|
985
|
-
...customActions
|
|
986
|
-
];
|
|
987
|
-
}
|
|
988
|
-
async destroy() {
|
|
989
|
-
if (this.destroyed) return;
|
|
990
|
-
this.destroyed = true;
|
|
991
|
-
this.inputDriver.destroy();
|
|
992
|
-
if (this.xvfbInstance) {
|
|
993
|
-
this.xvfbInstance.stop();
|
|
994
|
-
this.xvfbInstance = void 0;
|
|
995
|
-
}
|
|
996
|
-
if (this.xvfbCleanup) {
|
|
997
|
-
process.removeListener('exit', this.xvfbCleanup);
|
|
998
|
-
process.removeListener('SIGINT', this.xvfbCleanup);
|
|
999
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
1000
|
-
this.xvfbCleanup = void 0;
|
|
1001
|
-
}
|
|
1002
|
-
debugDevice('Computer device destroyed');
|
|
1003
|
-
}
|
|
1004
|
-
async url() {
|
|
1005
|
-
return '';
|
|
1006
|
-
}
|
|
1007
|
-
constructor(options){
|
|
1008
|
-
device_define_property(this, "interfaceType", 'computer');
|
|
1009
|
-
device_define_property(this, "options", void 0);
|
|
1010
|
-
device_define_property(this, "displayId", void 0);
|
|
1011
|
-
device_define_property(this, "displayGeometry", void 0);
|
|
1012
|
-
device_define_property(this, "description", void 0);
|
|
1013
|
-
device_define_property(this, "destroyed", false);
|
|
1014
|
-
device_define_property(this, "xvfbInstance", void 0);
|
|
1015
|
-
device_define_property(this, "xvfbCleanup", void 0);
|
|
1016
|
-
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
1017
|
-
getLibnut: ()=>device_libnut,
|
|
1018
|
-
useAppleScript: ()=>this.useAppleScript,
|
|
1019
|
-
sendKeyViaAppleScript,
|
|
1020
|
-
runPhasedScroll,
|
|
1021
|
-
debug: (message)=>debugDevice(message)
|
|
1022
|
-
}));
|
|
1023
|
-
device_define_property(this, "useAppleScript", void 0);
|
|
1024
|
-
device_define_property(this, "adminCheckCache", void 0);
|
|
1025
|
-
device_define_property(this, "uri", void 0);
|
|
1026
|
-
device_define_property(this, "inputPrimitives", {
|
|
1027
|
-
pointer: {
|
|
1028
|
-
tap: async ({ x, y }, opts)=>{
|
|
1029
|
-
const target = this.toGlobalPoint({
|
|
1030
|
-
x,
|
|
1031
|
-
y
|
|
1032
|
-
});
|
|
1033
|
-
const targetX = Math.round(target.x);
|
|
1034
|
-
const targetY = Math.round(target.y);
|
|
1035
|
-
const holdDuration = Math.max(0, Math.round(opts?.duration ?? CLICK_HOLD_DURATION));
|
|
1036
|
-
debugComputerInput('tap start %o', {
|
|
1037
|
-
local: {
|
|
1038
|
-
x,
|
|
1039
|
-
y
|
|
1040
|
-
},
|
|
1041
|
-
global: {
|
|
1042
|
-
x: targetX,
|
|
1043
|
-
y: targetY
|
|
1044
|
-
},
|
|
1045
|
-
holdDuration,
|
|
1046
|
-
displayId: this.displayId,
|
|
1047
|
-
displayGeometry: this.displayGeometry ? {
|
|
1048
|
-
screenIndex: this.displayGeometry.screenIndex,
|
|
1049
|
-
cgDisplayId: this.displayGeometry.cgDisplayId,
|
|
1050
|
-
bounds: this.displayGeometry.bounds
|
|
1051
|
-
} : void 0
|
|
1052
|
-
});
|
|
1053
|
-
const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
|
|
1054
|
-
await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
1055
|
-
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
|
|
1056
|
-
if (frontmostBefore && 'darwin' === process.platform) {
|
|
1057
|
-
await sleep(CLICK_FOCUS_SETTLE_DELAY);
|
|
1058
|
-
const frontmostAfter = readDarwinFrontmostApplication();
|
|
1059
|
-
const focusChanged = !!frontmostAfter && frontmostAfter.pid !== frontmostBefore.pid;
|
|
1060
|
-
debugComputerInput('tap focus check %o', {
|
|
1061
|
-
before: frontmostBefore,
|
|
1062
|
-
after: frontmostAfter,
|
|
1063
|
-
focusChanged
|
|
1064
|
-
});
|
|
1065
|
-
if (focusChanged) {
|
|
1066
|
-
this.inputDriver.moveMouse(targetX, targetY);
|
|
1067
|
-
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
},
|
|
1071
|
-
doubleClick: async ({ x, y })=>{
|
|
1072
|
-
const target = this.toGlobalPoint({
|
|
1073
|
-
x,
|
|
1074
|
-
y
|
|
1075
|
-
});
|
|
1076
|
-
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1077
|
-
this.inputDriver.mouseClick('left', true);
|
|
1078
|
-
},
|
|
1079
|
-
rightClick: async ({ x, y })=>{
|
|
1080
|
-
const target = this.toGlobalPoint({
|
|
1081
|
-
x,
|
|
1082
|
-
y
|
|
1083
|
-
});
|
|
1084
|
-
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1085
|
-
this.inputDriver.mouseClick('right');
|
|
1086
|
-
},
|
|
1087
|
-
hover: async ({ x, y })=>{
|
|
1088
|
-
const target = this.toGlobalPoint({
|
|
1089
|
-
x,
|
|
1090
|
-
y
|
|
1091
|
-
});
|
|
1092
|
-
await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
1093
|
-
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1094
|
-
},
|
|
1095
|
-
dragAndDrop: async (from, to)=>{
|
|
1096
|
-
const globalFrom = this.toGlobalPoint(from);
|
|
1097
|
-
const globalTo = this.toGlobalPoint(to);
|
|
1098
|
-
this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
|
|
1099
|
-
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1100
|
-
await this.inputDriver.delay(100);
|
|
1101
|
-
this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
|
|
1102
|
-
await this.inputDriver.delay(100);
|
|
1103
|
-
});
|
|
1104
|
-
}
|
|
1105
|
-
},
|
|
1106
|
-
keyboard: {
|
|
1107
|
-
typeText: async (value, opts)=>{
|
|
1108
|
-
const element = opts?.target;
|
|
1109
|
-
if (element) {
|
|
1110
|
-
const [x, y] = element.center;
|
|
1111
|
-
const target = this.toGlobalPoint({
|
|
1112
|
-
x,
|
|
1113
|
-
y
|
|
1114
|
-
});
|
|
1115
|
-
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1116
|
-
this.inputDriver.mouseClick('left');
|
|
1117
|
-
await this.inputDriver.delay(INPUT_FOCUS_DELAY);
|
|
1118
|
-
if (opts?.replace !== false) {
|
|
1119
|
-
await this.selectAllAndDelete();
|
|
1120
|
-
await this.inputDriver.delay(INPUT_CLEAR_DELAY);
|
|
1121
|
-
}
|
|
1122
|
-
}
|
|
1123
|
-
await this.smartTypeString(value);
|
|
1124
|
-
},
|
|
1125
|
-
keyboardPress: async (keyName, opts)=>{
|
|
1126
|
-
const target = opts?.target;
|
|
1127
|
-
if (target) {
|
|
1128
|
-
const [x, y] = target.center;
|
|
1129
|
-
const point = this.toGlobalPoint({
|
|
1130
|
-
x,
|
|
1131
|
-
y
|
|
1132
|
-
});
|
|
1133
|
-
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1134
|
-
this.inputDriver.mouseClick('left');
|
|
1135
|
-
await this.inputDriver.delay(50);
|
|
1136
|
-
}
|
|
1137
|
-
await this.pressKeyboardShortcut(keyName);
|
|
1138
|
-
},
|
|
1139
|
-
clearInput: async (target)=>{
|
|
1140
|
-
if (target) {
|
|
1141
|
-
const element = target;
|
|
1142
|
-
const [x, y] = element.center;
|
|
1143
|
-
const point = this.toGlobalPoint({
|
|
1144
|
-
x,
|
|
1145
|
-
y
|
|
1146
|
-
});
|
|
1147
|
-
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1148
|
-
this.inputDriver.mouseClick('left');
|
|
1149
|
-
await this.inputDriver.delay(100);
|
|
1150
|
-
}
|
|
1151
|
-
await this.selectAllAndDelete();
|
|
1152
|
-
await this.inputDriver.delay(50);
|
|
1153
|
-
}
|
|
1154
|
-
},
|
|
1155
|
-
scroll: {
|
|
1156
|
-
scroll: async (param)=>{
|
|
1157
|
-
await this.performScroll(param);
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
|
-
});
|
|
1161
|
-
this.options = options;
|
|
1162
|
-
this.displayId = options?.displayId;
|
|
1163
|
-
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
1164
|
-
}
|
|
1165
|
-
}
|
|
1166
|
-
function createPlatformActions() {
|
|
1167
|
-
return {
|
|
1168
|
-
ListDisplays: defineAction({
|
|
1169
|
-
name: 'ListDisplays',
|
|
1170
|
-
description: 'List all available displays/monitors',
|
|
1171
|
-
call: async ()=>await ComputerDevice.listDisplays()
|
|
1172
|
-
})
|
|
1173
|
-
};
|
|
1174
|
-
}
|
|
1175
|
-
function normalizeRdpHost(host) {
|
|
1176
|
-
const trimmed = host.trim();
|
|
1177
|
-
if (trimmed.length >= 2 && trimmed.startsWith('[') && trimmed.endsWith(']') && trimmed.includes(':')) return trimmed.slice(1, -1);
|
|
1178
|
-
return trimmed;
|
|
1179
|
-
}
|
|
1180
|
-
function formatRdpHost(host) {
|
|
1181
|
-
const normalizedHost = normalizeRdpHost(host);
|
|
1182
|
-
return normalizedHost.includes(':') ? `[${normalizedHost}]` : normalizedHost;
|
|
1183
|
-
}
|
|
1184
|
-
function formatRdpServerAddress(host, port) {
|
|
1185
|
-
return `${formatRdpHost(host)}:${port}`;
|
|
1186
|
-
}
|
|
1187
|
-
function normalizeRdpConnectionConfig(config) {
|
|
1188
|
-
return {
|
|
1189
|
-
...config,
|
|
1190
|
-
host: normalizeRdpHost(config.host),
|
|
1191
|
-
...config.localAddress ? {
|
|
1192
|
-
localAddress: config.localAddress.trim()
|
|
1193
|
-
} : {}
|
|
1194
|
-
};
|
|
1195
|
-
}
|
|
1196
|
-
const platformBinaryMap = {
|
|
1197
|
-
darwin: {
|
|
1198
|
-
directory: 'darwin',
|
|
1199
|
-
fileName: 'rdp-helper'
|
|
1200
|
-
},
|
|
1201
|
-
linux: {
|
|
1202
|
-
directory: 'linux',
|
|
1203
|
-
fileName: 'rdp-helper'
|
|
1204
|
-
},
|
|
1205
|
-
win32: {
|
|
1206
|
-
directory: 'win32',
|
|
1207
|
-
fileName: 'rdp-helper.exe'
|
|
1208
|
-
}
|
|
1209
|
-
};
|
|
1210
|
-
function getPlatformBinary(platform) {
|
|
1211
|
-
if (platform in platformBinaryMap) return platformBinaryMap[platform];
|
|
1212
|
-
}
|
|
1213
|
-
function currentDirname() {
|
|
1214
|
-
if ('undefined' != typeof __dirname) return __dirname;
|
|
1215
|
-
return dirname(fileURLToPath(import.meta.url));
|
|
1216
|
-
}
|
|
1217
|
-
function getRdpHelperBinaryPath() {
|
|
1218
|
-
const platformBinary = getPlatformBinary(process.platform);
|
|
1219
|
-
if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
|
|
1220
|
-
const hereDir = currentDirname();
|
|
1221
|
-
const candidateRoots = [
|
|
1222
|
-
external_node_path_resolve(hereDir, '../..'),
|
|
1223
|
-
external_node_path_resolve(hereDir, '../../..')
|
|
1224
|
-
];
|
|
1225
|
-
for (const root of candidateRoots){
|
|
1226
|
-
const binaryPath = external_node_path_resolve(root, 'bin', platformBinary.directory, platformBinary.fileName);
|
|
1227
|
-
if (existsSync(binaryPath)) return binaryPath;
|
|
1228
|
-
}
|
|
1229
|
-
throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
|
|
1230
|
-
}
|
|
1231
|
-
function backend_client_define_property(obj, key, value) {
|
|
1232
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
1233
|
-
value: value,
|
|
1234
|
-
enumerable: true,
|
|
1235
|
-
configurable: true,
|
|
1236
|
-
writable: true
|
|
1237
|
-
});
|
|
1238
|
-
else obj[key] = value;
|
|
1239
|
-
return obj;
|
|
1240
|
-
}
|
|
1241
|
-
const debug = getDebug('rdp:backend');
|
|
1242
|
-
const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
|
|
1243
|
-
const HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS = 50;
|
|
1244
|
-
const MAX_STDERR_CHARS = 16384;
|
|
1245
|
-
class HelperProcessRDPBackendClient {
|
|
1246
|
-
async connect(config) {
|
|
1247
|
-
this.fatalHelperError = void 0;
|
|
1248
|
-
const response = await this.send({
|
|
1249
|
-
type: 'connect',
|
|
1250
|
-
config: normalizeRdpConnectionConfig(config)
|
|
1251
|
-
});
|
|
1252
|
-
if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
|
|
1253
|
-
this.connected = true;
|
|
1254
|
-
this.fatalHelperError = void 0;
|
|
1255
|
-
return response.info;
|
|
1256
|
-
}
|
|
1257
|
-
async disconnect() {
|
|
1258
|
-
const child = this.child;
|
|
1259
|
-
if (!child) return;
|
|
1260
|
-
let disconnectError;
|
|
1261
|
-
if (this.connected && null === child.exitCode) try {
|
|
1262
|
-
const response = await this.send({
|
|
1263
|
-
type: 'disconnect'
|
|
1264
|
-
});
|
|
1265
|
-
this.expectOk(response, 'disconnect');
|
|
1266
|
-
} catch (error) {
|
|
1267
|
-
disconnectError = error instanceof Error ? error : new Error(String(error));
|
|
1268
|
-
}
|
|
1269
|
-
this.connected = false;
|
|
1270
|
-
this.fatalHelperError = void 0;
|
|
1271
|
-
await this.shutdownHelper();
|
|
1272
|
-
if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
|
|
1273
|
-
}
|
|
1274
|
-
async screenshotBase64() {
|
|
1275
|
-
const response = await this.send({
|
|
1276
|
-
type: 'screenshot'
|
|
1277
|
-
});
|
|
1278
|
-
if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
|
|
1279
|
-
return response.base64;
|
|
1280
|
-
}
|
|
1281
|
-
async size() {
|
|
1282
|
-
const response = await this.send({
|
|
1283
|
-
type: 'size'
|
|
1284
|
-
});
|
|
1285
|
-
if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
|
|
1286
|
-
return response.size;
|
|
1287
|
-
}
|
|
1288
|
-
async mouseMove(x, y) {
|
|
1289
|
-
const response = await this.send({
|
|
1290
|
-
type: 'mouseMove',
|
|
1291
|
-
x,
|
|
1292
|
-
y
|
|
1293
|
-
});
|
|
1294
|
-
this.expectOk(response, 'mouseMove');
|
|
1295
|
-
}
|
|
1296
|
-
async mouseButton(button, action) {
|
|
1297
|
-
const response = await this.send({
|
|
1298
|
-
type: 'mouseButton',
|
|
1299
|
-
button,
|
|
1300
|
-
action
|
|
1301
|
-
});
|
|
1302
|
-
this.expectOk(response, 'mouseButton');
|
|
1303
|
-
}
|
|
1304
|
-
async wheel(direction, amount, x, y) {
|
|
1305
|
-
const response = await this.send({
|
|
1306
|
-
type: 'wheel',
|
|
1307
|
-
direction,
|
|
1308
|
-
amount,
|
|
1309
|
-
x,
|
|
1310
|
-
y
|
|
1311
|
-
});
|
|
1312
|
-
this.expectOk(response, 'wheel');
|
|
1313
|
-
}
|
|
1314
|
-
async keyPress(keyName) {
|
|
1315
|
-
const response = await this.send({
|
|
1316
|
-
type: 'keyPress',
|
|
1317
|
-
keyName
|
|
1318
|
-
});
|
|
1319
|
-
this.expectOk(response, 'keyPress');
|
|
1320
|
-
}
|
|
1321
|
-
async typeText(text) {
|
|
1322
|
-
const response = await this.send({
|
|
1323
|
-
type: 'typeText',
|
|
1324
|
-
text
|
|
1325
|
-
});
|
|
1326
|
-
this.expectOk(response, 'typeText');
|
|
1327
|
-
}
|
|
1328
|
-
async clearInput() {
|
|
1329
|
-
const response = await this.send({
|
|
1330
|
-
type: 'clearInput'
|
|
1331
|
-
});
|
|
1332
|
-
this.expectOk(response, 'clearInput');
|
|
1333
|
-
}
|
|
1334
|
-
ensureHelperStarted() {
|
|
1335
|
-
if (this.child && null === this.child.exitCode) return this.child;
|
|
1336
|
-
const helperPath = this.resolveHelperPath();
|
|
1337
|
-
debug('starting rdp helper', {
|
|
1338
|
-
helperPath
|
|
1339
|
-
});
|
|
1340
|
-
const child = this.spawnFn(helperPath, [], {
|
|
1341
|
-
stdio: [
|
|
1342
|
-
'pipe',
|
|
1343
|
-
'pipe',
|
|
1344
|
-
'pipe'
|
|
1345
|
-
]
|
|
1346
|
-
});
|
|
1347
|
-
child.stdout.setEncoding('utf8');
|
|
1348
|
-
child.stderr.setEncoding('utf8');
|
|
1349
|
-
this.child = child;
|
|
1350
|
-
const diagnostics = {
|
|
1351
|
-
path: helperPath,
|
|
1352
|
-
pid: child.pid,
|
|
1353
|
-
stderrBuffer: ''
|
|
1354
|
-
};
|
|
1355
|
-
this.helperDiagnostics.set(child, diagnostics);
|
|
1356
|
-
debug('started rdp helper', {
|
|
1357
|
-
helperPath,
|
|
1358
|
-
pid: child.pid
|
|
1359
|
-
});
|
|
1360
|
-
this.stdoutReader = createInterface({
|
|
1361
|
-
input: child.stdout,
|
|
1362
|
-
crlfDelay: 1 / 0
|
|
1363
|
-
});
|
|
1364
|
-
this.stdoutReader.on('line', (line)=>{
|
|
1365
|
-
this.handleStdoutLine(child, line, diagnostics);
|
|
1366
|
-
});
|
|
1367
|
-
child.stderr.on('data', (chunk)=>{
|
|
1368
|
-
this.captureStderrChunk(chunk, diagnostics);
|
|
1369
|
-
});
|
|
1370
|
-
child.stdin.on('error', (error)=>{
|
|
1371
|
-
this.handleHelperStreamError(child, 'stdin', error, diagnostics);
|
|
1372
|
-
});
|
|
1373
|
-
child.stdout.on('error', (error)=>{
|
|
1374
|
-
this.handleHelperStreamError(child, 'stdout', error, diagnostics);
|
|
1375
|
-
});
|
|
1376
|
-
child.stderr.on('error', (error)=>{
|
|
1377
|
-
this.handleHelperStreamError(child, 'stderr', error, diagnostics);
|
|
1378
|
-
});
|
|
1379
|
-
child.on('exit', (code, signal)=>{
|
|
1380
|
-
diagnostics.exit = {
|
|
1381
|
-
code,
|
|
1382
|
-
signal
|
|
1383
|
-
};
|
|
1384
|
-
debug('rdp helper exited', {
|
|
1385
|
-
helperPath: diagnostics.path,
|
|
1386
|
-
pid: diagnostics.pid,
|
|
1387
|
-
code,
|
|
1388
|
-
signal
|
|
1389
|
-
});
|
|
1390
|
-
const helperError = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`, void 0, diagnostics);
|
|
1391
|
-
this.rejectPendingForChild(child, helperError);
|
|
1392
|
-
if (this.child === child) {
|
|
1393
|
-
this.connected = false;
|
|
1394
|
-
this.fatalHelperError = helperError;
|
|
1395
|
-
this.disposeReaders();
|
|
1396
|
-
this.child = void 0;
|
|
1397
|
-
}
|
|
1398
|
-
});
|
|
1399
|
-
child.on('error', (error)=>{
|
|
1400
|
-
const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`, void 0, diagnostics);
|
|
1401
|
-
this.rejectPendingForChild(child, helperError);
|
|
1402
|
-
if (this.child === child) {
|
|
1403
|
-
this.connected = false;
|
|
1404
|
-
this.fatalHelperError = helperError;
|
|
1405
|
-
this.disposeReaders();
|
|
1406
|
-
this.child = void 0;
|
|
1407
|
-
}
|
|
1408
|
-
});
|
|
1409
|
-
return child;
|
|
1410
|
-
}
|
|
1411
|
-
handleHelperStreamError(child, streamName, error, diagnostics) {
|
|
1412
|
-
const nodeError = error;
|
|
1413
|
-
const helperError = this.createHelperError(`RDP helper ${streamName} stream error: ${error.message}`, nodeError.code, diagnostics);
|
|
1414
|
-
this.rejectPendingForChild(child, helperError);
|
|
1415
|
-
if (this.child !== child) return;
|
|
1416
|
-
this.connected = false;
|
|
1417
|
-
this.fatalHelperError = helperError;
|
|
1418
|
-
if (null === child.exitCode) child.kill('SIGTERM');
|
|
1419
|
-
this.child = void 0;
|
|
1420
|
-
}
|
|
1421
|
-
handleStdoutLine(child, line, diagnostics) {
|
|
1422
|
-
if (!line.trim()) return;
|
|
1423
|
-
let parsed;
|
|
1424
|
-
try {
|
|
1425
|
-
parsed = JSON.parse(line);
|
|
1426
|
-
} catch (error) {
|
|
1427
|
-
const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`, void 0, diagnostics);
|
|
1428
|
-
this.rejectPendingForChild(child, protocolError);
|
|
1429
|
-
if (this.child === child) this.shutdownHelper(protocolError);
|
|
1430
|
-
return;
|
|
1431
|
-
}
|
|
1432
|
-
const pending = this.pending.get(parsed.id);
|
|
1433
|
-
if (!pending) return void debug('dropping response for unknown request id', parsed);
|
|
1434
|
-
this.pending.delete(parsed.id);
|
|
1435
|
-
if (parsed.ok) return void pending.resolve(parsed.payload);
|
|
1436
|
-
pending.reject(this.createHelperError(parsed.error.message, parsed.error.code, diagnostics));
|
|
1437
|
-
}
|
|
1438
|
-
captureStderrChunk(chunk, diagnostics) {
|
|
1439
|
-
const text = chunk.toString();
|
|
1440
|
-
if (!text.trim()) return;
|
|
1441
|
-
debug('rdp helper stderr', {
|
|
1442
|
-
helperPath: diagnostics.path,
|
|
1443
|
-
pid: diagnostics.pid,
|
|
1444
|
-
stderr: text.trim()
|
|
1445
|
-
});
|
|
1446
|
-
diagnostics.stderrBuffer += text;
|
|
1447
|
-
if (diagnostics.stderrBuffer.length > MAX_STDERR_CHARS) diagnostics.stderrBuffer = diagnostics.stderrBuffer.slice(-MAX_STDERR_CHARS);
|
|
1448
|
-
}
|
|
1449
|
-
async send(payload) {
|
|
1450
|
-
if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
|
|
1451
|
-
const child = this.ensureHelperStarted();
|
|
1452
|
-
if (null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
|
|
1453
|
-
const id = `req-${++this.nextRequestId}`;
|
|
1454
|
-
const diagnostics = this.helperDiagnostics.get(child);
|
|
1455
|
-
const request = {
|
|
1456
|
-
id,
|
|
1457
|
-
payload
|
|
1458
|
-
};
|
|
1459
|
-
return new Promise((resolve, reject)=>{
|
|
1460
|
-
this.pending.set(id, {
|
|
1461
|
-
resolve,
|
|
1462
|
-
reject,
|
|
1463
|
-
child
|
|
1464
|
-
});
|
|
1465
|
-
child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
|
|
1466
|
-
if (!error) return;
|
|
1467
|
-
this.pending.delete(id);
|
|
1468
|
-
const nodeError = error;
|
|
1469
|
-
const timer = setTimeout(()=>{
|
|
1470
|
-
reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`, nodeError.code, diagnostics));
|
|
1471
|
-
}, HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS);
|
|
1472
|
-
timer.unref?.();
|
|
1473
|
-
});
|
|
1474
|
-
});
|
|
1475
|
-
}
|
|
1476
|
-
expectOk(response, actionName) {
|
|
1477
|
-
if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
|
|
1478
|
-
}
|
|
1479
|
-
rejectPending(error) {
|
|
1480
|
-
for (const { reject } of this.pending.values())reject(error);
|
|
1481
|
-
this.pending.clear();
|
|
1482
|
-
}
|
|
1483
|
-
rejectPendingForChild(child, error) {
|
|
1484
|
-
for (const [id, pending] of this.pending)if (pending.child === child) {
|
|
1485
|
-
pending.reject(error);
|
|
1486
|
-
this.pending.delete(id);
|
|
1487
|
-
}
|
|
1488
|
-
}
|
|
1489
|
-
createHelperError(message, code, diagnostics) {
|
|
1490
|
-
const diagnosticParts = [
|
|
1491
|
-
diagnostics?.path ? `path=${diagnostics.path}` : void 0,
|
|
1492
|
-
'number' == typeof diagnostics?.pid ? `pid=${diagnostics.pid}` : void 0,
|
|
1493
|
-
diagnostics?.exit ? `exitCode=${diagnostics.exit.code}, signal=${diagnostics.exit.signal}` : void 0
|
|
1494
|
-
].filter(Boolean);
|
|
1495
|
-
const diagnosticsSuffix = diagnosticParts.length > 0 ? `\nHelper diagnostics: ${diagnosticParts.join(', ')}` : '';
|
|
1496
|
-
const stderrSummary = diagnostics?.stderrBuffer.trim();
|
|
1497
|
-
const stderrSuffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
|
|
1498
|
-
const error = new Error(`${message}${diagnosticsSuffix}${stderrSuffix}`);
|
|
1499
|
-
if (code) error.name = code;
|
|
1500
|
-
return error;
|
|
1501
|
-
}
|
|
1502
|
-
disposeReaders() {
|
|
1503
|
-
this.stdoutReader?.close();
|
|
1504
|
-
this.stdoutReader = void 0;
|
|
1505
|
-
}
|
|
1506
|
-
async shutdownHelper(rootError) {
|
|
1507
|
-
const child = this.child;
|
|
1508
|
-
this.child = void 0;
|
|
1509
|
-
this.disposeReaders();
|
|
1510
|
-
if (!child) return;
|
|
1511
|
-
this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
|
|
1512
|
-
if (null !== child.exitCode) return;
|
|
1513
|
-
child.stdin.end();
|
|
1514
|
-
const exited = Promise.race([
|
|
1515
|
-
once(child, 'exit'),
|
|
1516
|
-
new Promise((resolve)=>{
|
|
1517
|
-
setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
|
|
1518
|
-
})
|
|
1519
|
-
]);
|
|
1520
|
-
const result = await exited;
|
|
1521
|
-
if ('timeout' !== result) return;
|
|
1522
|
-
child.kill('SIGTERM');
|
|
1523
|
-
const terminated = Promise.race([
|
|
1524
|
-
once(child, 'exit'),
|
|
1525
|
-
new Promise((resolve)=>{
|
|
1526
|
-
setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
|
|
1527
|
-
})
|
|
1528
|
-
]);
|
|
1529
|
-
const terminateResult = await terminated;
|
|
1530
|
-
if ('timeout' !== terminateResult) return;
|
|
1531
|
-
child.kill('SIGKILL');
|
|
1532
|
-
await once(child, 'exit');
|
|
1533
|
-
}
|
|
1534
|
-
constructor(options){
|
|
1535
|
-
backend_client_define_property(this, "spawnFn", void 0);
|
|
1536
|
-
backend_client_define_property(this, "resolveHelperPath", void 0);
|
|
1537
|
-
backend_client_define_property(this, "child", void 0);
|
|
1538
|
-
backend_client_define_property(this, "stdoutReader", void 0);
|
|
1539
|
-
backend_client_define_property(this, "pending", new Map());
|
|
1540
|
-
backend_client_define_property(this, "helperDiagnostics", new WeakMap());
|
|
1541
|
-
backend_client_define_property(this, "nextRequestId", 0);
|
|
1542
|
-
backend_client_define_property(this, "connected", false);
|
|
1543
|
-
backend_client_define_property(this, "fatalHelperError", void 0);
|
|
1544
|
-
this.spawnFn = options?.spawnFn || spawn;
|
|
1545
|
-
const overridePath = options?.helperPath;
|
|
1546
|
-
this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
|
|
1547
|
-
}
|
|
1548
|
-
}
|
|
1549
|
-
function createDefaultRDPBackendClient() {
|
|
1550
|
-
return new HelperProcessRDPBackendClient();
|
|
1551
|
-
}
|
|
1552
|
-
function rdp_device_define_property(obj, key, value) {
|
|
1553
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
1554
|
-
value: value,
|
|
1555
|
-
enumerable: true,
|
|
1556
|
-
configurable: true,
|
|
1557
|
-
writable: true
|
|
1558
|
-
});
|
|
1559
|
-
else obj[key] = value;
|
|
1560
|
-
return obj;
|
|
1561
|
-
}
|
|
1562
|
-
const device_debug = getDebug('rdp:device');
|
|
1563
|
-
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
1564
|
-
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
1565
|
-
const SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
1566
|
-
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
1567
|
-
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
1568
|
-
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
1569
|
-
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
1570
|
-
const device_CLICK_HOLD_DURATION = 50;
|
|
1571
|
-
const DRAG_HOLD_DURATION = 100;
|
|
1572
|
-
const device_INPUT_FOCUS_DELAY = 300;
|
|
1573
|
-
const device_INPUT_CLEAR_DELAY = 150;
|
|
1574
|
-
const device_SCROLL_STEP_DELAY = 100;
|
|
1575
|
-
const device_SCROLL_COMPLETE_DELAY = 500;
|
|
1576
|
-
const DEFAULT_SCROLL_DISTANCE = 480;
|
|
1577
|
-
const device_DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
1578
|
-
const device_EDGE_SCROLL_STEPS = 10;
|
|
1579
|
-
const DEFAULT_SCROLL_STEP_AMOUNT = 120;
|
|
1580
|
-
class RDPDevice {
|
|
1581
|
-
describe() {
|
|
1582
|
-
const port = this.options.port || 3389;
|
|
1583
|
-
const server = formatRdpServerAddress(this.options.host, port);
|
|
1584
|
-
const username = this.options.username ? ` as ${this.options.username}` : '';
|
|
1585
|
-
const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
|
|
1586
|
-
return `RDP Device ${server}${username}${session}`;
|
|
1587
|
-
}
|
|
1588
|
-
async connect() {
|
|
1589
|
-
this.throwIfDestroyed();
|
|
1590
|
-
device_debug('connecting to rdp backend', {
|
|
1591
|
-
host: this.options.host,
|
|
1592
|
-
port: this.options.port,
|
|
1593
|
-
username: this.options.username
|
|
1594
|
-
});
|
|
1595
|
-
const { backend: _backend, customActions: _customActions, ...config } = this.options;
|
|
1596
|
-
this.connectionInfo = await this.backend.connect(config);
|
|
1597
|
-
this.cursorPosition = [
|
|
1598
|
-
Math.round(this.connectionInfo.size.width / 2),
|
|
1599
|
-
Math.round(this.connectionInfo.size.height / 2)
|
|
1600
|
-
];
|
|
1601
|
-
}
|
|
1602
|
-
async screenshotBase64() {
|
|
1603
|
-
this.assertConnected();
|
|
1604
|
-
return this.backend.screenshotBase64();
|
|
1605
|
-
}
|
|
1606
|
-
async size() {
|
|
1607
|
-
this.assertConnected();
|
|
1608
|
-
return this.backend.size();
|
|
1609
|
-
}
|
|
1610
|
-
async destroy() {
|
|
1611
|
-
if (this.destroyed) return;
|
|
1612
|
-
this.destroyed = true;
|
|
1613
|
-
this.connectionInfo = void 0;
|
|
1614
|
-
this.cursorPosition = void 0;
|
|
1615
|
-
await this.backend.disconnect();
|
|
1616
|
-
}
|
|
1617
|
-
actionSpace() {
|
|
1618
|
-
const defaultActions = [
|
|
1619
|
-
...defineActionsFromInputPrimitives(this.inputPrimitives),
|
|
1620
|
-
defineAction({
|
|
1621
|
-
name: 'ListDisplays',
|
|
1622
|
-
description: 'List all available displays/monitors',
|
|
1623
|
-
call: async ()=>{
|
|
1624
|
-
this.assertConnected();
|
|
1625
|
-
const size = await this.size();
|
|
1626
|
-
const server = this.connectionInfo?.server || formatRdpServerAddress(this.options.host, this.options.port || 3389);
|
|
1627
|
-
return [
|
|
1628
|
-
{
|
|
1629
|
-
id: this.connectionInfo?.sessionId || this.options.host,
|
|
1630
|
-
name: `RDP ${server} (${size.width}x${size.height})`,
|
|
1631
|
-
primary: true
|
|
1632
|
-
}
|
|
1633
|
-
];
|
|
1634
|
-
}
|
|
1635
|
-
})
|
|
1636
|
-
];
|
|
1637
|
-
return [
|
|
1638
|
-
...defaultActions,
|
|
1639
|
-
...this.options.customActions || []
|
|
1640
|
-
];
|
|
1641
|
-
}
|
|
1642
|
-
assertConnected() {
|
|
1643
|
-
this.throwIfDestroyed();
|
|
1644
|
-
if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
|
|
1645
|
-
}
|
|
1646
|
-
throwIfDestroyed() {
|
|
1647
|
-
if (this.destroyed) throw new Error('RDPDevice has been destroyed');
|
|
1648
|
-
}
|
|
1649
|
-
async moveToElement(element, options) {
|
|
1650
|
-
this.assertConnected();
|
|
1651
|
-
const targetX = Math.round(element.center[0]);
|
|
1652
|
-
const targetY = Math.round(element.center[1]);
|
|
1653
|
-
await this.movePointer(targetX, targetY, options);
|
|
1654
|
-
}
|
|
1655
|
-
async clearInput() {
|
|
1656
|
-
if (this.backend.clearInput) return void await this.backend.clearInput();
|
|
1657
|
-
await this.backend.keyPress('Control+A');
|
|
1658
|
-
await this.backend.keyPress('Backspace');
|
|
1659
|
-
}
|
|
1660
|
-
edgeScrollDirection(scrollType) {
|
|
1661
|
-
switch(scrollType){
|
|
1662
|
-
case 'scrollToTop':
|
|
1663
|
-
return 'up';
|
|
1664
|
-
case 'scrollToBottom':
|
|
1665
|
-
return 'down';
|
|
1666
|
-
case 'scrollToLeft':
|
|
1667
|
-
return 'left';
|
|
1668
|
-
case 'scrollToRight':
|
|
1669
|
-
return 'right';
|
|
1670
|
-
case 'singleAction':
|
|
1671
|
-
return 'down';
|
|
1672
|
-
default:
|
|
1673
|
-
throw new Error(`Unsupported scroll type: ${scrollType}`);
|
|
1674
|
-
}
|
|
1675
|
-
}
|
|
1676
|
-
defaultScrollDistance(direction) {
|
|
1677
|
-
const size = this.connectionInfo?.size;
|
|
1678
|
-
if (!size) return DEFAULT_SCROLL_DISTANCE;
|
|
1679
|
-
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
1680
|
-
const base = isHorizontal ? size.width : size.height;
|
|
1681
|
-
return Math.max(1, Math.round(base * device_DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
1682
|
-
}
|
|
1683
|
-
async movePointer(targetX, targetY, options) {
|
|
1684
|
-
this.assertConnected();
|
|
1685
|
-
const start = this.cursorPosition || [
|
|
1686
|
-
targetX,
|
|
1687
|
-
targetY
|
|
1688
|
-
];
|
|
1689
|
-
const steps = Math.max(1, options?.steps || 1);
|
|
1690
|
-
const stepDelayMs = options?.stepDelayMs || 0;
|
|
1691
|
-
for(let step = 1; step <= steps; step++){
|
|
1692
|
-
const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
|
|
1693
|
-
const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
|
|
1694
|
-
await this.backend.mouseMove(x, y);
|
|
1695
|
-
this.cursorPosition = [
|
|
1696
|
-
x,
|
|
1697
|
-
y
|
|
1698
|
-
];
|
|
1699
|
-
if (stepDelayMs > 0 && step < steps) await sleep(stepDelayMs);
|
|
1700
|
-
}
|
|
1701
|
-
if (options?.settleDelayMs) await sleep(options.settleDelayMs);
|
|
1702
|
-
}
|
|
1703
|
-
async performWheel(direction, amount, x, y) {
|
|
1704
|
-
let remaining = Math.abs(amount);
|
|
1705
|
-
if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
|
|
1706
|
-
while(remaining > 0){
|
|
1707
|
-
const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
|
|
1708
|
-
await this.backend.wheel(direction, chunk, x, y);
|
|
1709
|
-
remaining -= chunk;
|
|
1710
|
-
if (remaining > 0) await sleep(device_SCROLL_STEP_DELAY);
|
|
1711
|
-
}
|
|
1712
|
-
}
|
|
1713
|
-
constructor(options){
|
|
1714
|
-
rdp_device_define_property(this, "interfaceType", 'rdp');
|
|
1715
|
-
rdp_device_define_property(this, "options", void 0);
|
|
1716
|
-
rdp_device_define_property(this, "backend", void 0);
|
|
1717
|
-
rdp_device_define_property(this, "connectionInfo", void 0);
|
|
1718
|
-
rdp_device_define_property(this, "destroyed", false);
|
|
1719
|
-
rdp_device_define_property(this, "cursorPosition", void 0);
|
|
1720
|
-
rdp_device_define_property(this, "uri", void 0);
|
|
1721
|
-
rdp_device_define_property(this, "inputPrimitives", {
|
|
1722
|
-
pointer: {
|
|
1723
|
-
tap: async ({ x, y })=>{
|
|
1724
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1725
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1726
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1727
|
-
});
|
|
1728
|
-
await this.backend.mouseButton('left', 'down');
|
|
1729
|
-
await sleep(device_CLICK_HOLD_DURATION);
|
|
1730
|
-
await this.backend.mouseButton('left', 'up');
|
|
1731
|
-
},
|
|
1732
|
-
doubleClick: async ({ x, y })=>{
|
|
1733
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1734
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1735
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1736
|
-
});
|
|
1737
|
-
await this.backend.mouseButton('left', 'doubleClick');
|
|
1738
|
-
},
|
|
1739
|
-
rightClick: async ({ x, y })=>{
|
|
1740
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1741
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1742
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1743
|
-
});
|
|
1744
|
-
await this.backend.mouseButton('right', 'click');
|
|
1745
|
-
},
|
|
1746
|
-
hover: async ({ x, y })=>{
|
|
1747
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1748
|
-
steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1749
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
|
|
1750
|
-
settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
|
|
1751
|
-
});
|
|
1752
|
-
},
|
|
1753
|
-
dragAndDrop: async (from, to)=>{
|
|
1754
|
-
await this.movePointer(Math.round(from.x), Math.round(from.y), {
|
|
1755
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1756
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1757
|
-
});
|
|
1758
|
-
await this.backend.mouseButton('left', 'down');
|
|
1759
|
-
await sleep(DRAG_HOLD_DURATION);
|
|
1760
|
-
await this.movePointer(Math.round(to.x), Math.round(to.y), {
|
|
1761
|
-
steps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1762
|
-
stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
|
|
1763
|
-
});
|
|
1764
|
-
await sleep(DRAG_HOLD_DURATION);
|
|
1765
|
-
await this.backend.mouseButton('left', 'up');
|
|
1766
|
-
}
|
|
1767
|
-
},
|
|
1768
|
-
keyboard: {
|
|
1769
|
-
typeText: async (value, opts)=>{
|
|
1770
|
-
this.assertConnected();
|
|
1771
|
-
const target = opts?.target;
|
|
1772
|
-
if (target) {
|
|
1773
|
-
await this.inputPrimitives.pointer.tap({
|
|
1774
|
-
x: target.center[0],
|
|
1775
|
-
y: target.center[1]
|
|
1776
|
-
});
|
|
1777
|
-
await sleep(device_INPUT_FOCUS_DELAY);
|
|
1778
|
-
}
|
|
1779
|
-
if (opts?.replace !== false) {
|
|
1780
|
-
await this.clearInput();
|
|
1781
|
-
await sleep(device_INPUT_CLEAR_DELAY);
|
|
1782
|
-
}
|
|
1783
|
-
if (opts?.focusOnly || !value) return;
|
|
1784
|
-
await this.backend.typeText(value);
|
|
1785
|
-
},
|
|
1786
|
-
clearInput: async (target)=>{
|
|
1787
|
-
this.assertConnected();
|
|
1788
|
-
const element = target;
|
|
1789
|
-
if (element) {
|
|
1790
|
-
await this.inputPrimitives.pointer.tap({
|
|
1791
|
-
x: element.center[0],
|
|
1792
|
-
y: element.center[1]
|
|
1793
|
-
});
|
|
1794
|
-
await sleep(device_INPUT_FOCUS_DELAY);
|
|
1795
|
-
}
|
|
1796
|
-
await this.clearInput();
|
|
1797
|
-
await sleep(device_INPUT_CLEAR_DELAY);
|
|
1798
|
-
},
|
|
1799
|
-
keyboardPress: async (keyName, opts)=>{
|
|
1800
|
-
this.assertConnected();
|
|
1801
|
-
const target = opts?.target;
|
|
1802
|
-
if (target) await this.inputPrimitives.pointer.tap({
|
|
1803
|
-
x: target.center[0],
|
|
1804
|
-
y: target.center[1]
|
|
1805
|
-
});
|
|
1806
|
-
await this.backend.keyPress(keyName);
|
|
1807
|
-
}
|
|
1808
|
-
},
|
|
1809
|
-
scroll: {
|
|
1810
|
-
scroll: async (param)=>{
|
|
1811
|
-
this.assertConnected();
|
|
1812
|
-
const target = param.locate;
|
|
1813
|
-
if (target) await this.moveToElement(target, {
|
|
1814
|
-
steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1815
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1816
|
-
});
|
|
1817
|
-
if (param.scrollType && 'singleAction' !== param.scrollType) {
|
|
1818
|
-
const direction = this.edgeScrollDirection(param.scrollType);
|
|
1819
|
-
for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
|
|
1820
|
-
await sleep(device_SCROLL_COMPLETE_DELAY);
|
|
1821
|
-
return;
|
|
1822
|
-
}
|
|
1823
|
-
await this.performWheel(param.direction || 'down', param.distance || this.defaultScrollDistance(param.direction || 'down'), target?.center[0], target?.center[1]);
|
|
1824
|
-
await sleep(device_SCROLL_COMPLETE_DELAY);
|
|
1825
|
-
}
|
|
1826
|
-
}
|
|
1827
|
-
});
|
|
1828
|
-
const normalizedOptions = normalizeRdpConnectionConfig(options);
|
|
1829
|
-
this.options = {
|
|
1830
|
-
port: 3389,
|
|
1831
|
-
securityProtocol: 'auto',
|
|
1832
|
-
ignoreCertificate: false,
|
|
1833
|
-
...normalizedOptions
|
|
1834
|
-
};
|
|
1835
|
-
this.backend = options.backend || createDefaultRDPBackendClient();
|
|
1836
|
-
}
|
|
1837
|
-
}
|
|
1838
|
-
class ComputerAgent extends Agent {
|
|
1839
|
-
}
|
|
1840
|
-
function createLocalComputerDevice(opts) {
|
|
1841
|
-
return new ComputerDevice({
|
|
1842
|
-
displayId: opts?.displayId,
|
|
1843
|
-
customActions: opts?.customActions,
|
|
1844
|
-
keyboardDriver: opts?.keyboardDriver,
|
|
1845
|
-
headless: opts?.headless,
|
|
1846
|
-
xvfbResolution: opts?.xvfbResolution
|
|
1847
|
-
});
|
|
1848
|
-
}
|
|
1849
|
-
function createRDPComputerDevice(opts) {
|
|
1850
|
-
return new RDPDevice({
|
|
1851
|
-
host: opts.host,
|
|
1852
|
-
port: opts.port,
|
|
1853
|
-
username: opts.username,
|
|
1854
|
-
password: opts.password,
|
|
1855
|
-
domain: opts.domain,
|
|
1856
|
-
localAddress: opts.localAddress,
|
|
1857
|
-
adminSession: opts.adminSession,
|
|
1858
|
-
ignoreCertificate: opts.ignoreCertificate,
|
|
1859
|
-
securityProtocol: opts.securityProtocol,
|
|
1860
|
-
desktopWidth: opts.desktopWidth,
|
|
1861
|
-
desktopHeight: opts.desktopHeight,
|
|
1862
|
-
backend: opts.backend,
|
|
1863
|
-
customActions: opts.customActions
|
|
1864
|
-
});
|
|
1865
|
-
}
|
|
1866
|
-
async function agentForComputer(opts) {
|
|
1867
|
-
const device = createLocalComputerDevice(opts);
|
|
1868
|
-
await device.connect();
|
|
1869
|
-
return new ComputerAgent(device, opts);
|
|
1870
|
-
}
|
|
1871
|
-
const agentFromComputer = agentForComputer;
|
|
1872
|
-
async function agentForRDPComputer(opts) {
|
|
1873
|
-
const device = createRDPComputerDevice(opts);
|
|
1874
|
-
await device.connect();
|
|
1875
|
-
return new ComputerAgent(device, opts);
|
|
1876
|
-
}
|
|
1877
|
-
function mcp_tools_define_property(obj, key, value) {
|
|
1878
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
1879
|
-
value: value,
|
|
1880
|
-
enumerable: true,
|
|
1881
|
-
configurable: true,
|
|
1882
|
-
writable: true
|
|
1883
|
-
});
|
|
1884
|
-
else obj[key] = value;
|
|
1885
|
-
return obj;
|
|
1886
|
-
}
|
|
1887
|
-
const mcp_tools_debug = getDebug('mcp:computer-tools');
|
|
1888
|
-
const RDP_SECURITY_PROTOCOLS = [
|
|
1889
|
-
'auto',
|
|
1890
|
-
'tls',
|
|
1891
|
-
'nla',
|
|
1892
|
-
'rdp'
|
|
1893
|
-
];
|
|
1894
|
-
const computerInitArgShape = {
|
|
1895
|
-
displayId: z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
|
|
1896
|
-
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
1897
|
-
host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
1898
|
-
port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
1899
|
-
username: z.string().optional().describe('RDP username. Requires host.'),
|
|
1900
|
-
password: z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
|
|
1901
|
-
domain: z.string().optional().describe('RDP domain. Requires host.'),
|
|
1902
|
-
localAddress: z.string().optional().describe('Local source IP address for the RDP TCP connection. Requires host.'),
|
|
1903
|
-
adminSession: z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
|
|
1904
|
-
ignoreCertificate: z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
|
|
1905
|
-
securityProtocol: z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
|
|
1906
|
-
desktopWidth: z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
|
|
1907
|
-
desktopHeight: z.number().optional().describe('Remote desktop height in pixels. Requires host.'),
|
|
1908
|
-
...agentBehaviorInitArgShape
|
|
1909
|
-
};
|
|
1910
|
-
function adaptComputerInitArgs(extracted) {
|
|
1911
|
-
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
1912
|
-
if (extracted.host) {
|
|
1913
|
-
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
1914
|
-
const host = normalizeRdpHost(extracted.host);
|
|
1915
|
-
return {
|
|
1916
|
-
mode: 'rdp',
|
|
1917
|
-
...rdpFields,
|
|
1918
|
-
host
|
|
1919
|
-
};
|
|
1920
|
-
}
|
|
1921
|
-
return {
|
|
1922
|
-
mode: 'local',
|
|
1923
|
-
displayId: extracted.displayId,
|
|
1924
|
-
headless: extracted.headless,
|
|
1925
|
-
...extractAgentBehaviorInitArgs(extracted) ?? {}
|
|
1926
|
-
};
|
|
1927
|
-
}
|
|
1928
|
-
function describeConnectTarget(opts) {
|
|
1929
|
-
if (opts?.mode === 'rdp') {
|
|
1930
|
-
const target = opts.port ? formatRdpServerAddress(opts.host, opts.port) : formatRdpHost(opts.host);
|
|
1931
|
-
const userSuffix = opts.username ? ` as ${opts.username}` : '';
|
|
1932
|
-
return ` via RDP (${target}${userSuffix})`;
|
|
1933
|
-
}
|
|
1934
|
-
if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
|
|
1935
|
-
return ' (Primary display)';
|
|
1936
|
-
}
|
|
1937
|
-
function getCliReportSessionTarget(opts) {
|
|
1938
|
-
if (opts?.mode === 'rdp') return `rdp:${formatRdpHost(opts.host)}`;
|
|
1939
|
-
if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
|
|
1940
|
-
return 'primary';
|
|
1941
|
-
}
|
|
1942
|
-
class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
1943
|
-
getCliReportSessionName() {
|
|
1944
|
-
return 'midscene-computer';
|
|
1945
|
-
}
|
|
1946
|
-
createTemporaryDevice() {
|
|
1947
|
-
return new ComputerDevice({});
|
|
1948
|
-
}
|
|
1949
|
-
async ensureAgent(opts) {
|
|
1950
|
-
const nextSignature = getAgentInitArgsSignature(opts);
|
|
1951
|
-
if (this.agent && shouldRebuildAgentForInitArgs(this.lastInitArgsSignature, nextSignature)) {
|
|
1952
|
-
try {
|
|
1953
|
-
await this.agent.destroy?.();
|
|
1954
|
-
} catch (error) {
|
|
1955
|
-
mcp_tools_debug('Failed to destroy agent during cleanup:', error);
|
|
1956
|
-
}
|
|
1957
|
-
this.agent = void 0;
|
|
1958
|
-
}
|
|
1959
|
-
if (this.agent) return this.agent;
|
|
1960
|
-
const reportOptions = this.readCliReportAgentOptions();
|
|
1961
|
-
if (opts?.mode === 'rdp') {
|
|
1962
|
-
mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
|
|
1963
|
-
const { mode: _mode, ...rdpFields } = opts;
|
|
1964
|
-
const agent = await agentForRDPComputer({
|
|
1965
|
-
...rdpFields,
|
|
1966
|
-
...reportOptions ?? {}
|
|
1967
|
-
});
|
|
1968
|
-
this.agent = agent;
|
|
1969
|
-
this.lastInitArgsSignature = nextSignature;
|
|
1970
|
-
return agent;
|
|
1971
|
-
}
|
|
1972
|
-
const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
|
|
1973
|
-
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
1974
|
-
mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
1975
|
-
const agentOpts = {
|
|
1976
|
-
...displayId ? {
|
|
1977
|
-
displayId
|
|
1978
|
-
} : {},
|
|
1979
|
-
...void 0 !== headless ? {
|
|
1980
|
-
headless
|
|
1981
|
-
} : {},
|
|
1982
|
-
...extractAgentBehaviorInitArgs(opts) ?? {},
|
|
1983
|
-
...reportOptions ?? {}
|
|
1984
|
-
};
|
|
1985
|
-
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
1986
|
-
this.agent = agent;
|
|
1987
|
-
this.lastInitArgsSignature = nextSignature;
|
|
1988
|
-
return agent;
|
|
1989
|
-
}
|
|
1990
|
-
preparePlatformTools() {
|
|
1991
|
-
return [
|
|
1992
|
-
{
|
|
1993
|
-
name: 'computer_connect',
|
|
1994
|
-
description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/localAddress/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
|
|
1995
|
-
schema: this.getAgentInitArgSchema(),
|
|
1996
|
-
cli: this.getAgentInitArgCliMetadata(),
|
|
1997
|
-
handler: async (args)=>{
|
|
1998
|
-
const initArgs = this.extractAgentInitParam(args);
|
|
1999
|
-
const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
|
|
2000
|
-
this.commitCliReportSession(reportSession);
|
|
2001
|
-
if (this.agent) {
|
|
2002
|
-
try {
|
|
2003
|
-
await this.agent.destroy?.();
|
|
2004
|
-
} catch (error) {
|
|
2005
|
-
mcp_tools_debug('Failed to destroy agent during connect:', error);
|
|
2006
|
-
}
|
|
2007
|
-
this.agent = void 0;
|
|
2008
|
-
this.lastInitArgsSignature = void 0;
|
|
2009
|
-
}
|
|
2010
|
-
const agent = await this.ensureAgent(initArgs);
|
|
2011
|
-
const screenshot = await agent.interface.screenshotBase64();
|
|
2012
|
-
return {
|
|
2013
|
-
content: [
|
|
2014
|
-
{
|
|
2015
|
-
type: 'text',
|
|
2016
|
-
text: `Connected to computer${describeConnectTarget(initArgs)}`
|
|
2017
|
-
},
|
|
2018
|
-
...this.buildScreenshotContent(screenshot)
|
|
2019
|
-
]
|
|
2020
|
-
};
|
|
2021
|
-
}
|
|
2022
|
-
},
|
|
2023
|
-
{
|
|
2024
|
-
name: 'computer_disconnect',
|
|
2025
|
-
description: 'Disconnect from computer and release resources',
|
|
2026
|
-
schema: {},
|
|
2027
|
-
handler: this.createDisconnectHandler('computer')
|
|
2028
|
-
},
|
|
2029
|
-
{
|
|
2030
|
-
name: 'computer_list_displays',
|
|
2031
|
-
description: 'List all available displays/monitors',
|
|
2032
|
-
schema: {},
|
|
2033
|
-
handler: async ()=>{
|
|
2034
|
-
const displays = await ComputerDevice.listDisplays();
|
|
2035
|
-
return {
|
|
2036
|
-
content: [
|
|
2037
|
-
{
|
|
2038
|
-
type: 'text',
|
|
2039
|
-
text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
|
|
2040
|
-
}
|
|
2041
|
-
]
|
|
2042
|
-
};
|
|
2043
|
-
}
|
|
2044
|
-
}
|
|
2045
|
-
];
|
|
2046
|
-
}
|
|
2047
|
-
constructor(...args){
|
|
2048
|
-
super(...args), mcp_tools_define_property(this, "lastInitArgsSignature", void 0), mcp_tools_define_property(this, "initArgSpec", {
|
|
2049
|
-
namespace: 'computer',
|
|
2050
|
-
shape: computerInitArgShape,
|
|
2051
|
-
cli: {
|
|
2052
|
-
preferBareKeys: true
|
|
2053
|
-
},
|
|
2054
|
-
adapt: (extracted)=>adaptComputerInitArgs(extracted)
|
|
2055
|
-
});
|
|
2056
|
-
}
|
|
2057
|
-
}
|
|
2058
|
-
class ComputerMCPServer extends BaseMCPServer {
|
|
2059
|
-
createToolsManager() {
|
|
2060
|
-
return new ComputerMidsceneTools();
|
|
2061
|
-
}
|
|
2062
|
-
constructor(toolsManager){
|
|
2063
|
-
super({
|
|
2064
|
-
name: '@midscene/computer-mcp',
|
|
2065
|
-
version: "1.9.8",
|
|
2066
|
-
description: 'Control the computer desktop using natural language commands'
|
|
2067
|
-
}, toolsManager);
|
|
2068
|
-
}
|
|
2069
|
-
}
|
|
2070
|
-
function mcpServerForAgent(agent) {
|
|
2071
|
-
return createMCPServerLauncher({
|
|
2072
|
-
agent,
|
|
2073
|
-
platformName: 'Computer',
|
|
2074
|
-
ToolsManagerClass: ComputerMidsceneTools,
|
|
2075
|
-
MCPServerClass: ComputerMCPServer
|
|
2076
|
-
});
|
|
2077
|
-
}
|
|
2078
|
-
async function mcpKitForAgent(agent) {
|
|
2079
|
-
const toolsManager = new ComputerMidsceneTools();
|
|
2080
|
-
const computerAgent = agent instanceof ComputerAgent ? agent : agent;
|
|
2081
|
-
toolsManager.setAgent(computerAgent);
|
|
2082
|
-
await toolsManager.initTools();
|
|
2083
|
-
return {
|
|
2084
|
-
description: 'Midscene MCP Kit for computer desktop automation',
|
|
2085
|
-
tools: toolsManager.getToolDefinitions()
|
|
2086
|
-
};
|
|
2087
|
-
}
|
|
2088
|
-
export { ComputerMCPServer, mcpKitForAgent, mcpServerForAgent };
|