@midscene/computer 1.9.8-beta-20260618091332.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2088 +0,0 @@
1
- import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
2
- import { Agent } from "@midscene/core/agent";
3
- import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
4
- import { chmodSync, existsSync, statSync } from "node:fs";
5
- import { createRequire } from "node:module";
6
- import { dirname, resolve as external_node_path_resolve } from "node:path";
7
- import { fileURLToPath } from "node:url";
8
- import { defineAction, defineActionsFromInputPrimitives } from "@midscene/core/device";
9
- import { sleep } from "@midscene/core/utils";
10
- import { createImgBase64ByFormat } from "@midscene/shared/img";
11
- import { getDebug } from "@midscene/shared/logger";
12
- import screenshot_desktop from "screenshot-desktop";
13
- import node_assert from "node:assert";
14
- import { once } from "node:events";
15
- import { createInterface } from "node:readline";
16
- import { z } from "@midscene/core";
17
- import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/mcp/agent-behavior-init-args";
18
- import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
19
- function _define_property(obj, key, value) {
20
- if (key in obj) Object.defineProperty(obj, key, {
21
- value: value,
22
- enumerable: true,
23
- configurable: true,
24
- writable: true
25
- });
26
- else obj[key] = value;
27
- return obj;
28
- }
29
- class ComputerInputDriver {
30
- destroy() {
31
- if (this.destroyed) return;
32
- this.destroyed = true;
33
- this.rejectPendingInputDelays();
34
- }
35
- getScreenSize() {
36
- return this.getLibnutOrThrow('getScreenSize').getScreenSize();
37
- }
38
- getMousePos() {
39
- return this.getLibnutOrThrow('getMousePos').getMousePos();
40
- }
41
- moveMouse(x, y) {
42
- this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
43
- }
44
- focusActiveWindow() {
45
- const lib = this.getLibnutOrThrow('focusActiveWindow');
46
- if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
47
- try {
48
- const handle = lib.getActiveWindow();
49
- if (!handle) return false;
50
- lib.focusWindow(handle);
51
- return true;
52
- } catch (error) {
53
- this.options.debug(`focusActiveWindow failed: ${error}`);
54
- return false;
55
- }
56
- }
57
- getActiveWindowRect() {
58
- const lib = this.getLibnutOrThrow('getActiveWindowRect');
59
- if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
60
- try {
61
- const handle = lib.getActiveWindow();
62
- if (!handle) return null;
63
- const rect = lib.getWindowRect(handle);
64
- if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
65
- return rect;
66
- } catch (error) {
67
- this.options.debug(`getActiveWindowRect failed: ${error}`);
68
- return null;
69
- }
70
- }
71
- mouseClick(button, double) {
72
- const lib = this.getLibnutOrThrow('mouseClick');
73
- if (void 0 !== double) lib.mouseClick(button, double);
74
- else if (void 0 !== button) lib.mouseClick(button);
75
- else lib.mouseClick();
76
- }
77
- mouseToggle(state, button = 'left') {
78
- this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
79
- }
80
- scrollMouse(x, y) {
81
- this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
82
- }
83
- async emitScrollDetents(dx, dy, detents, delayMs) {
84
- this.assertActive('emitScrollDetents');
85
- for(let i = 0; i < detents; i++){
86
- this.scrollMouse(dx, dy);
87
- if (i < detents - 1) await this.delay(delayMs);
88
- }
89
- }
90
- keyTap(key, modifiers) {
91
- const lib = this.getLibnutOrThrow('keyTap');
92
- if (void 0 !== modifiers) lib.keyTap(key, modifiers);
93
- else lib.keyTap(key);
94
- }
95
- sendKeyViaAppleScript(key, modifiers = []) {
96
- this.assertActive('sendKeyViaAppleScript');
97
- this.options.sendKeyViaAppleScript(key, modifiers);
98
- }
99
- sendKey(key, modifiers = []) {
100
- if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
101
- if (modifiers.length > 0) this.keyTap(key, modifiers);
102
- else this.keyTap(key);
103
- }
104
- runPhasedScroll(direction, pixels, steps) {
105
- this.assertActive('runPhasedScroll');
106
- return this.options.runPhasedScroll(direction, pixels, steps);
107
- }
108
- async delay(ms) {
109
- this.assertActive('delay');
110
- return new Promise((resolve, reject)=>{
111
- const waitRef = {
112
- timeoutId: setTimeout(()=>{
113
- this.pendingInputDelayWaits.delete(waitRef);
114
- try {
115
- this.assertActive('delay');
116
- resolve();
117
- } catch (error) {
118
- reject(error);
119
- }
120
- }, ms),
121
- reject
122
- };
123
- this.pendingInputDelayWaits.add(waitRef);
124
- });
125
- }
126
- async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
127
- const currentPos = this.getMousePos();
128
- for(let i = 1; i <= steps; i++){
129
- const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
130
- const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
131
- this.moveMouse(stepX, stepY);
132
- await this.delay(stepDelay);
133
- }
134
- }
135
- async withMouseButton(button, run) {
136
- this.mouseToggle('down', button);
137
- try {
138
- return await run();
139
- } finally{
140
- this.releaseMouseButton(button);
141
- }
142
- }
143
- getLibnutOrThrow(methodName) {
144
- this.assertActive(methodName);
145
- const libnut = this.options.getLibnut();
146
- node_assert(libnut, 'libnut not initialized');
147
- return libnut;
148
- }
149
- assertActive(methodName) {
150
- if (this.destroyed) throw this.createDestroyedError(methodName);
151
- }
152
- createDestroyedError(methodName) {
153
- return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
154
- }
155
- releaseMouseButton(button) {
156
- try {
157
- const libnut = this.options.getLibnut();
158
- node_assert(libnut, 'libnut not initialized');
159
- libnut.mouseToggle('up', button);
160
- } catch (error) {
161
- this.options.debug(`Failed to release mouse button ${button}: ${error}`);
162
- }
163
- }
164
- rejectPendingInputDelays() {
165
- const error = this.createDestroyedError('in-flight input');
166
- for (const waitRef of this.pendingInputDelayWaits){
167
- clearTimeout(waitRef.timeoutId);
168
- waitRef.reject(error);
169
- }
170
- this.pendingInputDelayWaits.clear();
171
- }
172
- constructor(options){
173
- _define_property(this, "options", void 0);
174
- _define_property(this, "destroyed", void 0);
175
- _define_property(this, "pendingInputDelayWaits", void 0);
176
- this.options = options;
177
- this.destroyed = false;
178
- this.pendingInputDelayWaits = new Set();
179
- }
180
- }
181
- const debugXvfb = getDebug('computer:xvfb');
182
- function checkXvfbInstalled() {
183
- try {
184
- execSync('which Xvfb', {
185
- stdio: 'ignore'
186
- });
187
- return true;
188
- } catch {
189
- return false;
190
- }
191
- }
192
- function findAvailableDisplay(startFrom = 99) {
193
- for(let n = startFrom; n < startFrom + 100; n++)if (!existsSync(`/tmp/.X${n}-lock`)) return n;
194
- throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
195
- }
196
- function startXvfb(options) {
197
- const resolution = options?.resolution || '1920x1080x24';
198
- const displayNum = options?.displayNumber ?? findAvailableDisplay();
199
- const display = `:${displayNum}`;
200
- return new Promise((resolve, reject)=>{
201
- debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
202
- const xvfbProcess = spawn('Xvfb', [
203
- display,
204
- '-screen',
205
- '0',
206
- resolution,
207
- '-ac',
208
- '-nolisten',
209
- 'tcp'
210
- ], {
211
- stdio: 'ignore'
212
- });
213
- let settled = false;
214
- xvfbProcess.on('error', (err)=>{
215
- if (!settled) {
216
- settled = true;
217
- reject(new Error(`Failed to start Xvfb: ${err.message}`));
218
- }
219
- });
220
- xvfbProcess.on('exit', (code)=>{
221
- if (!settled) {
222
- settled = true;
223
- reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
224
- }
225
- });
226
- const instance = {
227
- process: xvfbProcess,
228
- display,
229
- stop () {
230
- try {
231
- xvfbProcess.kill('SIGTERM');
232
- } catch {}
233
- }
234
- };
235
- setTimeout(()=>{
236
- if (!settled) {
237
- settled = true;
238
- debugXvfb(`Xvfb started on display ${display}`);
239
- resolve(instance);
240
- }
241
- }, 500);
242
- });
243
- }
244
- function needsXvfb(explicitOpt) {
245
- if ('linux' !== process.platform) return false;
246
- return true === explicitOpt;
247
- }
248
- function device_define_property(obj, key, value) {
249
- if (key in obj) Object.defineProperty(obj, key, {
250
- value: value,
251
- enumerable: true,
252
- configurable: true,
253
- writable: true
254
- });
255
- else obj[key] = value;
256
- return obj;
257
- }
258
- const SMOOTH_MOVE_STEPS_TAP = 8;
259
- const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
260
- const SMOOTH_MOVE_DELAY_TAP = 8;
261
- const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
262
- const MOUSE_MOVE_EFFECT_WAIT = 300;
263
- const CLICK_SETTLE_DELAY = 50;
264
- const CLICK_HOLD_DURATION = 100;
265
- const CLICK_FOCUS_SETTLE_DELAY = 120;
266
- const INPUT_FOCUS_DELAY = 300;
267
- const INPUT_CLEAR_DELAY = 150;
268
- const SCROLL_STEP_DELAY = 100;
269
- const SCROLL_COMPLETE_DELAY = 500;
270
- const EDGE_SCROLL_TOTAL_PX = 50000;
271
- const EDGE_SCROLL_STEPS = 400;
272
- const PHASED_PIXELS_PER_STEP = 30;
273
- const PHASED_MIN_STEPS = 10;
274
- const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
275
- const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
276
- const LIBNUT_FALLBACK_MAX_DETENTS = 200;
277
- const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
278
- const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
279
- const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
280
- const EDGE_SCROLL_SPEC = {
281
- scrollToTop: {
282
- direction: 'up',
283
- key: 'home',
284
- libnut: [
285
- 0,
286
- 1
287
- ]
288
- },
289
- scrollToBottom: {
290
- direction: 'down',
291
- key: 'end',
292
- libnut: [
293
- 0,
294
- -1
295
- ]
296
- },
297
- scrollToLeft: {
298
- direction: 'left',
299
- key: 'home',
300
- libnut: [
301
- -1,
302
- 0
303
- ]
304
- },
305
- scrollToRight: {
306
- direction: 'right',
307
- key: 'end',
308
- libnut: [
309
- 1,
310
- 0
311
- ]
312
- }
313
- };
314
- const APPLESCRIPT_KEY_CODE_MAP = {
315
- return: 36,
316
- enter: 36,
317
- tab: 48,
318
- space: 49,
319
- backspace: 51,
320
- delete: 51,
321
- escape: 53,
322
- forwarddelete: 117,
323
- left: 123,
324
- right: 124,
325
- down: 125,
326
- up: 126,
327
- home: 115,
328
- end: 119,
329
- pageup: 116,
330
- pagedown: 121,
331
- f1: 122,
332
- f2: 120,
333
- f3: 99,
334
- f4: 118,
335
- f5: 96,
336
- f6: 97,
337
- f7: 98,
338
- f8: 100,
339
- f9: 101,
340
- f10: 109,
341
- f11: 103,
342
- f12: 111
343
- };
344
- const APPLESCRIPT_MODIFIER_MAP = {
345
- command: 'command down',
346
- cmd: 'command down',
347
- control: 'control down',
348
- ctrl: 'control down',
349
- shift: 'shift down',
350
- alt: 'option down',
351
- option: 'option down',
352
- meta: 'command down'
353
- };
354
- function sendKeyViaAppleScript(key, modifiers = []) {
355
- const lowerKey = key.toLowerCase();
356
- const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
357
- const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
358
- const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
359
- let script;
360
- if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
361
- else {
362
- const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
363
- script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
364
- }
365
- debugDevice('sendKeyViaAppleScript', {
366
- key,
367
- modifiers,
368
- script
369
- });
370
- execFileSync("osascript", [
371
- '-e',
372
- script
373
- ]);
374
- }
375
- let device_libnut = null;
376
- let libnutLoadError = null;
377
- async function getLibnut() {
378
- if (device_libnut) return device_libnut;
379
- if (libnutLoadError) throw libnutLoadError;
380
- try {
381
- const require = createRequire(import.meta.url);
382
- const libnutModule = require('@computer-use/libnut/dist/import_libnut');
383
- device_libnut = libnutModule.libnut;
384
- if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
385
- return device_libnut;
386
- } catch (error) {
387
- libnutLoadError = error;
388
- throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
389
- }
390
- }
391
- const debugDevice = getDebug('computer:device');
392
- const warnDevice = getDebug('computer:device', {
393
- console: true
394
- });
395
- const debugComputerInput = getDebug('computer:input', {
396
- console: true
397
- });
398
- const WINDOWS_UIPI_DOC_URL = 'https://midscenejs.com/computer-getting-started#windows-clicks-have-no-effect-on-some-apps';
399
- function resolvePackageRoot(helperName) {
400
- const require = createRequire(import.meta.url);
401
- let pkgRoot = null;
402
- try {
403
- pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
404
- } catch {
405
- const hereDir = dirname(fileURLToPath(import.meta.url));
406
- for (const candidate of [
407
- external_node_path_resolve(hereDir, '..'),
408
- external_node_path_resolve(hereDir, '../..')
409
- ])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
410
- pkgRoot = candidate;
411
- break;
412
- }
413
- }
414
- if (!pkgRoot) {
415
- debugDevice(`${helperName}: cannot locate @midscene/computer package root`);
416
- return null;
417
- }
418
- return pkgRoot;
419
- }
420
- let phasedScrollBinaryPath;
421
- function getPhasedScrollBinary() {
422
- if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
423
- if ('darwin' !== process.platform) {
424
- phasedScrollBinaryPath = null;
425
- return null;
426
- }
427
- const pkgRoot = resolvePackageRoot('phased-scroll');
428
- if (!pkgRoot) {
429
- phasedScrollBinaryPath = null;
430
- return null;
431
- }
432
- const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
433
- if (!existsSync(binPath)) {
434
- debugDevice('phased-scroll binary not found at', binPath);
435
- phasedScrollBinaryPath = null;
436
- return null;
437
- }
438
- try {
439
- const st = statSync(binPath);
440
- if ((73 & st.mode) === 0) {
441
- chmodSync(binPath, 493);
442
- debugDevice('phased-scroll: restored executable bit on', binPath);
443
- }
444
- } catch (err) {
445
- debugDevice('phased-scroll: chmod self-heal failed', err);
446
- }
447
- phasedScrollBinaryPath = binPath;
448
- return binPath;
449
- }
450
- let displayInfoBinaryPath;
451
- function getDisplayInfoBinary() {
452
- if (void 0 !== displayInfoBinaryPath) return displayInfoBinaryPath;
453
- if ('darwin' !== process.platform) {
454
- displayInfoBinaryPath = null;
455
- return null;
456
- }
457
- const pkgRoot = resolvePackageRoot('display-info');
458
- if (!pkgRoot) {
459
- displayInfoBinaryPath = null;
460
- return null;
461
- }
462
- const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/display-info');
463
- if (!existsSync(binPath)) {
464
- debugDevice('display-info binary not found at', binPath);
465
- displayInfoBinaryPath = null;
466
- return null;
467
- }
468
- displayInfoBinaryPath = binPath;
469
- return binPath;
470
- }
471
- function isFiniteNumber(value) {
472
- return 'number' == typeof value && Number.isFinite(value);
473
- }
474
- function isDarwinDisplayGeometry(value) {
475
- if (!value || 'object' != typeof value) return false;
476
- const candidate = value;
477
- return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
478
- }
479
- function readDarwinDisplayGeometries() {
480
- const bin = getDisplayInfoBinary();
481
- if (!bin) return [];
482
- try {
483
- const output = execFileSync(bin, [], {
484
- encoding: 'utf8',
485
- stdio: [
486
- 'ignore',
487
- 'pipe',
488
- 'pipe'
489
- ]
490
- });
491
- const parsed = JSON.parse(output);
492
- return Array.isArray(parsed.displays) ? parsed.displays.filter(isDarwinDisplayGeometry) : [];
493
- } catch (error) {
494
- debugDevice('display-info helper failed:', error);
495
- return [];
496
- }
497
- }
498
- function readDarwinFrontmostApplication() {
499
- try {
500
- const output = execFileSync("osascript", [
501
- '-e',
502
- 'tell application "System Events"\nset frontApp to first application process whose frontmost is true\nreturn (unix id of frontApp as string) & " " & (name of frontApp as string)\nend tell'
503
- ]).toString().trim();
504
- const [pidText, ...nameParts] = output.split('\t');
505
- const pid = Number(pidText);
506
- if (!Number.isInteger(pid) || pid <= 0) return;
507
- return {
508
- pid,
509
- name: nameParts.join('\t')
510
- };
511
- } catch (error) {
512
- debugDevice('Failed to read macOS frontmost application:', error);
513
- return;
514
- }
515
- }
516
- async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
517
- await inputDriver.delay(CLICK_SETTLE_DELAY);
518
- const current = inputDriver.getMousePos();
519
- debugComputerInput('tap mouse moved %o', {
520
- reason,
521
- target: {
522
- x: targetX,
523
- y: targetY
524
- },
525
- current,
526
- drift: {
527
- x: current.x - targetX,
528
- y: current.y - targetY
529
- }
530
- });
531
- await inputDriver.withMouseButton('left', async ()=>{
532
- debugComputerInput('tap mouse down %o', {
533
- reason
534
- });
535
- await inputDriver.delay(holdDuration);
536
- });
537
- debugComputerInput('tap mouse up %o', {
538
- reason
539
- });
540
- }
541
- function resolveDarwinDisplayGeometryFromList(displayId, displays) {
542
- if (!displays.length) return;
543
- const screenIndex = void 0 === displayId || '' === displayId ? 0 : Number(displayId);
544
- if (!Number.isInteger(screenIndex) || screenIndex < 0) return void debugDevice('Invalid macOS display id for display geometry:', displayId);
545
- if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
546
- return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
547
- }
548
- function resolveDisplayGeometry(displayId) {
549
- if ('darwin' !== process.platform) return;
550
- return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
551
- }
552
- function mapDisplayLocalPointToGlobal(point, geometry) {
553
- if (!geometry) return point;
554
- return {
555
- x: point.x + geometry.bounds.x,
556
- y: point.y + geometry.bounds.y
557
- };
558
- }
559
- let phasedScrollExecWarned = false;
560
- function runPhasedScroll(direction, pixels, steps) {
561
- const bin = getPhasedScrollBinary();
562
- if (!bin) return false;
563
- try {
564
- const res = spawnSync(bin, [
565
- direction,
566
- String(Math.max(1, Math.round(pixels))),
567
- String(steps)
568
- ], {
569
- stdio: 'ignore'
570
- });
571
- if (0 === res.status) return true;
572
- if (!phasedScrollExecWarned) {
573
- phasedScrollExecWarned = true;
574
- const hint = null === res.status ? `signal ${res.signal ?? 'unknown'}; the binary may not be executable (npm tarball extraction can drop the +x bit) or may be blocked by quarantine. Try: chmod +x "${bin}"` : 'this usually means Accessibility permission has not been granted to the host process (System Settings → Privacy & Security → Accessibility)';
575
- console.warn(`[@midscene/computer] phased-scroll helper failed (exit=${res.status}, signal=${res.signal ?? 'none'}); falling back to keyboard/libnut. ${hint}`);
576
- }
577
- debugDevice('phased-scroll exited non-zero', res.status, res.signal, res.error);
578
- return false;
579
- } catch (err) {
580
- if (!phasedScrollExecWarned) {
581
- phasedScrollExecWarned = true;
582
- console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
583
- }
584
- debugDevice('phased-scroll spawn failed', err);
585
- return false;
586
- }
587
- }
588
- const KEY_NAME_MAP = {
589
- windows: 'win',
590
- win: 'win',
591
- ctrl: 'control',
592
- esc: 'escape',
593
- del: 'delete',
594
- ins: 'insert',
595
- pgup: 'pageup',
596
- pgdn: 'pagedown',
597
- arrowup: 'up',
598
- arrowdown: 'down',
599
- arrowleft: 'left',
600
- arrowright: 'right',
601
- volumedown: 'audio_vol_down',
602
- volumeup: 'audio_vol_up',
603
- mediavolumedown: 'audio_vol_down',
604
- mediavolumeup: 'audio_vol_up',
605
- mute: 'audio_mute',
606
- mediamute: 'audio_mute',
607
- mediaplay: 'audio_play',
608
- mediapause: 'audio_pause',
609
- mediaplaypause: 'audio_play',
610
- mediastop: 'audio_stop',
611
- medianexttrack: 'audio_next',
612
- mediaprevioustrack: 'audio_prev',
613
- medianext: 'audio_next',
614
- mediaprev: 'audio_prev'
615
- };
616
- const PRIMARY_KEY_MAP = {
617
- command: 'cmd',
618
- cmd: 'cmd',
619
- meta: 'meta',
620
- control: 'control',
621
- ctrl: 'control',
622
- shift: 'shift',
623
- alt: 'alt',
624
- option: 'alt'
625
- };
626
- function normalizeKeyName(key) {
627
- const lowerKey = key.toLowerCase();
628
- return KEY_NAME_MAP[lowerKey] || lowerKey;
629
- }
630
- function normalizePrimaryKey(key) {
631
- const lowerKey = key.toLowerCase();
632
- if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
633
- return KEY_NAME_MAP[lowerKey] || lowerKey;
634
- }
635
- class ComputerDevice {
636
- describe() {
637
- return this.description || 'Computer Device';
638
- }
639
- static async listDisplays() {
640
- try {
641
- const displays = await screenshot_desktop.listDisplays();
642
- return displays.map((d)=>({
643
- id: String(d.id),
644
- name: d.name || `Display ${d.id}`,
645
- primary: d.primary || false
646
- }));
647
- } catch (error) {
648
- debugDevice(`Failed to list displays: ${error}`);
649
- return [];
650
- }
651
- }
652
- async connect() {
653
- debugDevice('Connecting to computer device');
654
- try {
655
- const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
656
- if (needsXvfb(headless)) {
657
- if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
658
- this.xvfbInstance = await startXvfb({
659
- resolution: this.options?.xvfbResolution
660
- });
661
- process.env.DISPLAY = this.xvfbInstance.display;
662
- debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
663
- this.xvfbCleanup = ()=>{
664
- if (this.xvfbInstance) {
665
- this.xvfbInstance.stop();
666
- this.xvfbInstance = void 0;
667
- }
668
- };
669
- process.on('exit', this.xvfbCleanup);
670
- process.on('SIGINT', this.xvfbCleanup);
671
- process.on('SIGTERM', this.xvfbCleanup);
672
- }
673
- device_libnut = await getLibnut();
674
- this.displayGeometry = resolveDisplayGeometry(this.displayId);
675
- const size = await this.size();
676
- const displays = await ComputerDevice.listDisplays();
677
- const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
678
- this.description = `
679
- Type: Computer
680
- Platform: ${process.platform}
681
- Display: ${this.displayId || 'Primary'}
682
- Screen Size: ${size.width}x${size.height}
683
- Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
684
- `;
685
- debugDevice('Computer device connected', this.description);
686
- await this.healthCheck();
687
- } catch (error) {
688
- if (this.xvfbInstance) {
689
- this.xvfbInstance.stop();
690
- this.xvfbInstance = void 0;
691
- }
692
- debugDevice(`Failed to connect: ${error}`);
693
- throw new Error(`Unable to connect to computer device: ${error}`);
694
- }
695
- }
696
- async healthCheck() {
697
- console.log('[HealthCheck] Starting health check...');
698
- console.log("[HealthCheck] @midscene/computer v1.9.8-beta-20260618091332.0");
699
- console.log('[HealthCheck] Taking screenshot...');
700
- const screenshotTimeout = 15000;
701
- let timeoutId;
702
- const timeoutPromise = new Promise((_, reject)=>{
703
- timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
704
- });
705
- const base64 = await Promise.race([
706
- this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
707
- timeoutPromise
708
- ]);
709
- console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
710
- console.log('[HealthCheck] Moving mouse...');
711
- const startPos = this.inputDriver.getMousePos();
712
- console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
713
- const offsetX = Math.floor(40 * Math.random()) + 10;
714
- const offsetY = Math.floor(40 * Math.random()) + 10;
715
- const targetX = startPos.x + offsetX;
716
- const targetY = startPos.y + offsetY;
717
- console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
718
- this.inputDriver.moveMouse(targetX, targetY);
719
- await sleep(50);
720
- const movedPos = this.inputDriver.getMousePos();
721
- console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
722
- const deltaX = Math.abs(movedPos.x - targetX);
723
- const deltaY = Math.abs(movedPos.y - targetY);
724
- if (deltaX > 5 || deltaY > 5) {
725
- const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
726
- warnDevice(msg);
727
- }
728
- if ('win32' === process.platform && !this.isRunningAsAdmin()) {
729
- const hint = [
730
- 'Heads-up: Midscene is not running as Administrator.',
731
- 'If clicks or key presses have no effect while the cursor still moves to the right position,',
732
- `see the Windows permission troubleshooting guide: ${WINDOWS_UIPI_DOC_URL}`
733
- ].join(' ');
734
- warnDevice(`[HealthCheck] ${hint}`);
735
- }
736
- this.inputDriver.moveMouse(startPos.x, startPos.y);
737
- console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
738
- console.log('[HealthCheck] Listing monitors...');
739
- const displays = await ComputerDevice.listDisplays();
740
- if (displays.length > 0) {
741
- console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
742
- for (const display of displays){
743
- const primaryTag = display.primary ? ' (primary)' : '';
744
- console.log(`[HealthCheck] - id=${display.id}, name=${display.name}${primaryTag}`);
745
- }
746
- } else console.log('[HealthCheck] No monitors detected');
747
- console.log('[HealthCheck] Health check passed');
748
- }
749
- isRunningAsAdmin() {
750
- if ('win32' !== process.platform) return false;
751
- if (void 0 !== this.adminCheckCache) return this.adminCheckCache;
752
- try {
753
- execSync('net session', {
754
- stdio: 'pipe'
755
- });
756
- this.adminCheckCache = true;
757
- } catch {
758
- this.adminCheckCache = false;
759
- }
760
- return this.adminCheckCache;
761
- }
762
- async screenshotBase64() {
763
- if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
764
- debugDevice('Taking screenshot', {
765
- displayId: this.displayId
766
- });
767
- const options = {
768
- format: 'png'
769
- };
770
- if (void 0 !== this.displayId) if ('darwin' === process.platform) {
771
- const screenIndex = Number(this.displayId);
772
- if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
773
- } else options.screen = this.displayId;
774
- debugDevice('Screenshot options', options);
775
- const MAX_ATTEMPTS = 3;
776
- const RETRY_DELAY_MS = 300;
777
- let lastRawMessage = '';
778
- for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
779
- const buffer = await screenshot_desktop(options);
780
- if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
781
- return createImgBase64ByFormat('png', buffer.toString('base64'));
782
- } catch (error) {
783
- lastRawMessage = error instanceof Error ? error.message : String(error);
784
- const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
785
- const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
786
- debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
787
- if (!willRetry) break;
788
- await sleep(RETRY_DELAY_MS);
789
- }
790
- if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
791
-
792
- Please follow these steps:
793
- 1. Open System Settings > Privacy & Security > Screen Recording
794
- 2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
795
- 3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
796
-
797
- Original error: ${lastRawMessage}`);
798
- throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
799
- }
800
- async size() {
801
- if (this.displayGeometry) return {
802
- width: Math.round(this.displayGeometry.bounds.width),
803
- height: Math.round(this.displayGeometry.bounds.height)
804
- };
805
- try {
806
- const screenSize = this.inputDriver.getScreenSize();
807
- return {
808
- width: screenSize.width,
809
- height: screenSize.height
810
- };
811
- } catch (error) {
812
- debugDevice(`Failed to get screen size: ${error}`);
813
- throw new Error(`Failed to get screen size: ${error}`);
814
- }
815
- }
816
- toGlobalPoint(point) {
817
- return mapDisplayLocalPointToGlobal(point, this.displayGeometry);
818
- }
819
- async typeViaClipboard(text) {
820
- debugDevice('Using clipboard to input text', {
821
- textLength: text.length,
822
- preview: text.substring(0, 20)
823
- });
824
- const clipboardy = await import("clipboardy");
825
- const oldClipboard = await clipboardy.default.read().catch(()=>'');
826
- try {
827
- await clipboardy.default.write(text);
828
- await this.inputDriver.delay(50);
829
- if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
830
- 'command'
831
- ]);
832
- else {
833
- const modifier = 'darwin' === process.platform ? 'command' : 'control';
834
- this.inputDriver.keyTap('v', [
835
- modifier
836
- ]);
837
- }
838
- await this.inputDriver.delay(100);
839
- } finally{
840
- if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
841
- debugDevice('Failed to restore clipboard content');
842
- });
843
- }
844
- }
845
- async smartTypeString(text) {
846
- await this.typeViaClipboard(text);
847
- }
848
- async selectAllAndDelete() {
849
- if (this.useAppleScript) {
850
- this.inputDriver.sendKeyViaAppleScript('a', [
851
- 'command'
852
- ]);
853
- await this.inputDriver.delay(50);
854
- this.inputDriver.sendKeyViaAppleScript('backspace', []);
855
- return;
856
- }
857
- const modifier = 'darwin' === process.platform ? 'command' : 'control';
858
- this.inputDriver.keyTap('a', [
859
- modifier
860
- ]);
861
- await this.inputDriver.delay(50);
862
- this.inputDriver.keyTap('backspace');
863
- }
864
- async pressKeyboardShortcut(keyName) {
865
- const keys = keyName.split('+');
866
- const modifiers = keys.slice(0, -1).map(normalizeKeyName);
867
- const key = normalizePrimaryKey(keys[keys.length - 1]);
868
- debugDevice('KeyboardPress', {
869
- original: keyName,
870
- key,
871
- modifiers,
872
- driver: this.useAppleScript ? "applescript" : 'libnut'
873
- });
874
- this.inputDriver.sendKey(key, modifiers);
875
- }
876
- resolveUntargetedScrollPoint(screenSize) {
877
- if ('win32' === process.platform) {
878
- const activeWindowRect = this.inputDriver.getActiveWindowRect();
879
- if (activeWindowRect) return {
880
- x: activeWindowRect.x + activeWindowRect.width / 2,
881
- y: activeWindowRect.y + activeWindowRect.height / 2
882
- };
883
- }
884
- return this.toGlobalPoint({
885
- x: screenSize.width / 2,
886
- y: screenSize.height / 2
887
- });
888
- }
889
- async moveMouseToScrollTarget(param) {
890
- if (param.locate) {
891
- const element = param.locate;
892
- const [x, y] = element.center;
893
- const point = this.toGlobalPoint({
894
- x,
895
- y
896
- });
897
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
898
- return;
899
- }
900
- const screenSize = await this.size();
901
- if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
902
- const point = this.resolveUntargetedScrollPoint(screenSize);
903
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
904
- await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
905
- return screenSize;
906
- }
907
- async performScroll(param) {
908
- let screenSize = await this.moveMouseToScrollTarget(param);
909
- const scrollType = param?.scrollType;
910
- const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
911
- if (edgeSpec) {
912
- if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
913
- if (this.useAppleScript) {
914
- this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
915
- await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
916
- return;
917
- }
918
- const [ux, uy] = edgeSpec.libnut;
919
- await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
920
- return;
921
- }
922
- if ('singleAction' === scrollType || !scrollType) {
923
- const direction = param?.direction || 'down';
924
- const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
925
- const isHorizontal = 'left' === direction || 'right' === direction;
926
- let distance = param?.distance ?? void 0;
927
- if (!distance) {
928
- screenSize ??= await this.size();
929
- const base = isHorizontal ? screenSize.width : screenSize.height;
930
- distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
931
- }
932
- if (isKnownDirection) {
933
- const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
934
- if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
935
- }
936
- if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
937
- if (!screenSize) screenSize = await this.size();
938
- const pages = Math.max(1, Math.round(distance / screenSize.height));
939
- const key = 'up' === direction ? 'pageup' : 'pagedown';
940
- for(let i = 0; i < pages; i++){
941
- this.inputDriver.sendKeyViaAppleScript(key);
942
- await this.inputDriver.delay(SCROLL_STEP_DELAY);
943
- }
944
- await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
945
- return;
946
- }
947
- const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
948
- const directionUnit = {
949
- up: [
950
- 0,
951
- 1
952
- ],
953
- down: [
954
- 0,
955
- -1
956
- ],
957
- left: [
958
- -1,
959
- 0
960
- ],
961
- right: [
962
- 1,
963
- 0
964
- ]
965
- };
966
- const [ux, uy] = directionUnit[direction] || [
967
- 0,
968
- -1
969
- ];
970
- await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
971
- await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
972
- return;
973
- }
974
- throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
975
- }
976
- actionSpace() {
977
- const defaultActions = [
978
- ...defineActionsFromInputPrimitives(this.inputPrimitives)
979
- ];
980
- const platformActions = Object.values(createPlatformActions());
981
- const customActions = this.options?.customActions || [];
982
- return [
983
- ...defaultActions,
984
- ...platformActions,
985
- ...customActions
986
- ];
987
- }
988
- async destroy() {
989
- if (this.destroyed) return;
990
- this.destroyed = true;
991
- this.inputDriver.destroy();
992
- if (this.xvfbInstance) {
993
- this.xvfbInstance.stop();
994
- this.xvfbInstance = void 0;
995
- }
996
- if (this.xvfbCleanup) {
997
- process.removeListener('exit', this.xvfbCleanup);
998
- process.removeListener('SIGINT', this.xvfbCleanup);
999
- process.removeListener('SIGTERM', this.xvfbCleanup);
1000
- this.xvfbCleanup = void 0;
1001
- }
1002
- debugDevice('Computer device destroyed');
1003
- }
1004
- async url() {
1005
- return '';
1006
- }
1007
- constructor(options){
1008
- device_define_property(this, "interfaceType", 'computer');
1009
- device_define_property(this, "options", void 0);
1010
- device_define_property(this, "displayId", void 0);
1011
- device_define_property(this, "displayGeometry", void 0);
1012
- device_define_property(this, "description", void 0);
1013
- device_define_property(this, "destroyed", false);
1014
- device_define_property(this, "xvfbInstance", void 0);
1015
- device_define_property(this, "xvfbCleanup", void 0);
1016
- device_define_property(this, "inputDriver", new ComputerInputDriver({
1017
- getLibnut: ()=>device_libnut,
1018
- useAppleScript: ()=>this.useAppleScript,
1019
- sendKeyViaAppleScript,
1020
- runPhasedScroll,
1021
- debug: (message)=>debugDevice(message)
1022
- }));
1023
- device_define_property(this, "useAppleScript", void 0);
1024
- device_define_property(this, "adminCheckCache", void 0);
1025
- device_define_property(this, "uri", void 0);
1026
- device_define_property(this, "inputPrimitives", {
1027
- pointer: {
1028
- tap: async ({ x, y }, opts)=>{
1029
- const target = this.toGlobalPoint({
1030
- x,
1031
- y
1032
- });
1033
- const targetX = Math.round(target.x);
1034
- const targetY = Math.round(target.y);
1035
- const holdDuration = Math.max(0, Math.round(opts?.duration ?? CLICK_HOLD_DURATION));
1036
- debugComputerInput('tap start %o', {
1037
- local: {
1038
- x,
1039
- y
1040
- },
1041
- global: {
1042
- x: targetX,
1043
- y: targetY
1044
- },
1045
- holdDuration,
1046
- displayId: this.displayId,
1047
- displayGeometry: this.displayGeometry ? {
1048
- screenIndex: this.displayGeometry.screenIndex,
1049
- cgDisplayId: this.displayGeometry.cgDisplayId,
1050
- bounds: this.displayGeometry.bounds
1051
- } : void 0
1052
- });
1053
- const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
1054
- await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
1055
- await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
1056
- if (frontmostBefore && 'darwin' === process.platform) {
1057
- await sleep(CLICK_FOCUS_SETTLE_DELAY);
1058
- const frontmostAfter = readDarwinFrontmostApplication();
1059
- const focusChanged = !!frontmostAfter && frontmostAfter.pid !== frontmostBefore.pid;
1060
- debugComputerInput('tap focus check %o', {
1061
- before: frontmostBefore,
1062
- after: frontmostAfter,
1063
- focusChanged
1064
- });
1065
- if (focusChanged) {
1066
- this.inputDriver.moveMouse(targetX, targetY);
1067
- await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
1068
- }
1069
- }
1070
- },
1071
- doubleClick: async ({ x, y })=>{
1072
- const target = this.toGlobalPoint({
1073
- x,
1074
- y
1075
- });
1076
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1077
- this.inputDriver.mouseClick('left', true);
1078
- },
1079
- rightClick: async ({ x, y })=>{
1080
- const target = this.toGlobalPoint({
1081
- x,
1082
- y
1083
- });
1084
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1085
- this.inputDriver.mouseClick('right');
1086
- },
1087
- hover: async ({ x, y })=>{
1088
- const target = this.toGlobalPoint({
1089
- x,
1090
- y
1091
- });
1092
- await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
1093
- await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
1094
- },
1095
- dragAndDrop: async (from, to)=>{
1096
- const globalFrom = this.toGlobalPoint(from);
1097
- const globalTo = this.toGlobalPoint(to);
1098
- this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
1099
- await this.inputDriver.withMouseButton('left', async ()=>{
1100
- await this.inputDriver.delay(100);
1101
- this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
1102
- await this.inputDriver.delay(100);
1103
- });
1104
- }
1105
- },
1106
- keyboard: {
1107
- typeText: async (value, opts)=>{
1108
- const element = opts?.target;
1109
- if (element) {
1110
- const [x, y] = element.center;
1111
- const target = this.toGlobalPoint({
1112
- x,
1113
- y
1114
- });
1115
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1116
- this.inputDriver.mouseClick('left');
1117
- await this.inputDriver.delay(INPUT_FOCUS_DELAY);
1118
- if (opts?.replace !== false) {
1119
- await this.selectAllAndDelete();
1120
- await this.inputDriver.delay(INPUT_CLEAR_DELAY);
1121
- }
1122
- }
1123
- await this.smartTypeString(value);
1124
- },
1125
- keyboardPress: async (keyName, opts)=>{
1126
- const target = opts?.target;
1127
- if (target) {
1128
- const [x, y] = target.center;
1129
- const point = this.toGlobalPoint({
1130
- x,
1131
- y
1132
- });
1133
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
1134
- this.inputDriver.mouseClick('left');
1135
- await this.inputDriver.delay(50);
1136
- }
1137
- await this.pressKeyboardShortcut(keyName);
1138
- },
1139
- clearInput: async (target)=>{
1140
- if (target) {
1141
- const element = target;
1142
- const [x, y] = element.center;
1143
- const point = this.toGlobalPoint({
1144
- x,
1145
- y
1146
- });
1147
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
1148
- this.inputDriver.mouseClick('left');
1149
- await this.inputDriver.delay(100);
1150
- }
1151
- await this.selectAllAndDelete();
1152
- await this.inputDriver.delay(50);
1153
- }
1154
- },
1155
- scroll: {
1156
- scroll: async (param)=>{
1157
- await this.performScroll(param);
1158
- }
1159
- }
1160
- });
1161
- this.options = options;
1162
- this.displayId = options?.displayId;
1163
- this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
1164
- }
1165
- }
1166
- function createPlatformActions() {
1167
- return {
1168
- ListDisplays: defineAction({
1169
- name: 'ListDisplays',
1170
- description: 'List all available displays/monitors',
1171
- call: async ()=>await ComputerDevice.listDisplays()
1172
- })
1173
- };
1174
- }
1175
- function normalizeRdpHost(host) {
1176
- const trimmed = host.trim();
1177
- if (trimmed.length >= 2 && trimmed.startsWith('[') && trimmed.endsWith(']') && trimmed.includes(':')) return trimmed.slice(1, -1);
1178
- return trimmed;
1179
- }
1180
- function formatRdpHost(host) {
1181
- const normalizedHost = normalizeRdpHost(host);
1182
- return normalizedHost.includes(':') ? `[${normalizedHost}]` : normalizedHost;
1183
- }
1184
- function formatRdpServerAddress(host, port) {
1185
- return `${formatRdpHost(host)}:${port}`;
1186
- }
1187
- function normalizeRdpConnectionConfig(config) {
1188
- return {
1189
- ...config,
1190
- host: normalizeRdpHost(config.host),
1191
- ...config.localAddress ? {
1192
- localAddress: config.localAddress.trim()
1193
- } : {}
1194
- };
1195
- }
1196
- const platformBinaryMap = {
1197
- darwin: {
1198
- directory: 'darwin',
1199
- fileName: 'rdp-helper'
1200
- },
1201
- linux: {
1202
- directory: 'linux',
1203
- fileName: 'rdp-helper'
1204
- },
1205
- win32: {
1206
- directory: 'win32',
1207
- fileName: 'rdp-helper.exe'
1208
- }
1209
- };
1210
- function getPlatformBinary(platform) {
1211
- if (platform in platformBinaryMap) return platformBinaryMap[platform];
1212
- }
1213
- function currentDirname() {
1214
- if ('undefined' != typeof __dirname) return __dirname;
1215
- return dirname(fileURLToPath(import.meta.url));
1216
- }
1217
- function getRdpHelperBinaryPath() {
1218
- const platformBinary = getPlatformBinary(process.platform);
1219
- if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
1220
- const hereDir = currentDirname();
1221
- const candidateRoots = [
1222
- external_node_path_resolve(hereDir, '../..'),
1223
- external_node_path_resolve(hereDir, '../../..')
1224
- ];
1225
- for (const root of candidateRoots){
1226
- const binaryPath = external_node_path_resolve(root, 'bin', platformBinary.directory, platformBinary.fileName);
1227
- if (existsSync(binaryPath)) return binaryPath;
1228
- }
1229
- throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
1230
- }
1231
- function backend_client_define_property(obj, key, value) {
1232
- if (key in obj) Object.defineProperty(obj, key, {
1233
- value: value,
1234
- enumerable: true,
1235
- configurable: true,
1236
- writable: true
1237
- });
1238
- else obj[key] = value;
1239
- return obj;
1240
- }
1241
- const debug = getDebug('rdp:backend');
1242
- const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
1243
- const HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS = 50;
1244
- const MAX_STDERR_CHARS = 16384;
1245
- class HelperProcessRDPBackendClient {
1246
- async connect(config) {
1247
- this.fatalHelperError = void 0;
1248
- const response = await this.send({
1249
- type: 'connect',
1250
- config: normalizeRdpConnectionConfig(config)
1251
- });
1252
- if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
1253
- this.connected = true;
1254
- this.fatalHelperError = void 0;
1255
- return response.info;
1256
- }
1257
- async disconnect() {
1258
- const child = this.child;
1259
- if (!child) return;
1260
- let disconnectError;
1261
- if (this.connected && null === child.exitCode) try {
1262
- const response = await this.send({
1263
- type: 'disconnect'
1264
- });
1265
- this.expectOk(response, 'disconnect');
1266
- } catch (error) {
1267
- disconnectError = error instanceof Error ? error : new Error(String(error));
1268
- }
1269
- this.connected = false;
1270
- this.fatalHelperError = void 0;
1271
- await this.shutdownHelper();
1272
- if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
1273
- }
1274
- async screenshotBase64() {
1275
- const response = await this.send({
1276
- type: 'screenshot'
1277
- });
1278
- if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
1279
- return response.base64;
1280
- }
1281
- async size() {
1282
- const response = await this.send({
1283
- type: 'size'
1284
- });
1285
- if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
1286
- return response.size;
1287
- }
1288
- async mouseMove(x, y) {
1289
- const response = await this.send({
1290
- type: 'mouseMove',
1291
- x,
1292
- y
1293
- });
1294
- this.expectOk(response, 'mouseMove');
1295
- }
1296
- async mouseButton(button, action) {
1297
- const response = await this.send({
1298
- type: 'mouseButton',
1299
- button,
1300
- action
1301
- });
1302
- this.expectOk(response, 'mouseButton');
1303
- }
1304
- async wheel(direction, amount, x, y) {
1305
- const response = await this.send({
1306
- type: 'wheel',
1307
- direction,
1308
- amount,
1309
- x,
1310
- y
1311
- });
1312
- this.expectOk(response, 'wheel');
1313
- }
1314
- async keyPress(keyName) {
1315
- const response = await this.send({
1316
- type: 'keyPress',
1317
- keyName
1318
- });
1319
- this.expectOk(response, 'keyPress');
1320
- }
1321
- async typeText(text) {
1322
- const response = await this.send({
1323
- type: 'typeText',
1324
- text
1325
- });
1326
- this.expectOk(response, 'typeText');
1327
- }
1328
- async clearInput() {
1329
- const response = await this.send({
1330
- type: 'clearInput'
1331
- });
1332
- this.expectOk(response, 'clearInput');
1333
- }
1334
- ensureHelperStarted() {
1335
- if (this.child && null === this.child.exitCode) return this.child;
1336
- const helperPath = this.resolveHelperPath();
1337
- debug('starting rdp helper', {
1338
- helperPath
1339
- });
1340
- const child = this.spawnFn(helperPath, [], {
1341
- stdio: [
1342
- 'pipe',
1343
- 'pipe',
1344
- 'pipe'
1345
- ]
1346
- });
1347
- child.stdout.setEncoding('utf8');
1348
- child.stderr.setEncoding('utf8');
1349
- this.child = child;
1350
- const diagnostics = {
1351
- path: helperPath,
1352
- pid: child.pid,
1353
- stderrBuffer: ''
1354
- };
1355
- this.helperDiagnostics.set(child, diagnostics);
1356
- debug('started rdp helper', {
1357
- helperPath,
1358
- pid: child.pid
1359
- });
1360
- this.stdoutReader = createInterface({
1361
- input: child.stdout,
1362
- crlfDelay: 1 / 0
1363
- });
1364
- this.stdoutReader.on('line', (line)=>{
1365
- this.handleStdoutLine(child, line, diagnostics);
1366
- });
1367
- child.stderr.on('data', (chunk)=>{
1368
- this.captureStderrChunk(chunk, diagnostics);
1369
- });
1370
- child.stdin.on('error', (error)=>{
1371
- this.handleHelperStreamError(child, 'stdin', error, diagnostics);
1372
- });
1373
- child.stdout.on('error', (error)=>{
1374
- this.handleHelperStreamError(child, 'stdout', error, diagnostics);
1375
- });
1376
- child.stderr.on('error', (error)=>{
1377
- this.handleHelperStreamError(child, 'stderr', error, diagnostics);
1378
- });
1379
- child.on('exit', (code, signal)=>{
1380
- diagnostics.exit = {
1381
- code,
1382
- signal
1383
- };
1384
- debug('rdp helper exited', {
1385
- helperPath: diagnostics.path,
1386
- pid: diagnostics.pid,
1387
- code,
1388
- signal
1389
- });
1390
- const helperError = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`, void 0, diagnostics);
1391
- this.rejectPendingForChild(child, helperError);
1392
- if (this.child === child) {
1393
- this.connected = false;
1394
- this.fatalHelperError = helperError;
1395
- this.disposeReaders();
1396
- this.child = void 0;
1397
- }
1398
- });
1399
- child.on('error', (error)=>{
1400
- const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`, void 0, diagnostics);
1401
- this.rejectPendingForChild(child, helperError);
1402
- if (this.child === child) {
1403
- this.connected = false;
1404
- this.fatalHelperError = helperError;
1405
- this.disposeReaders();
1406
- this.child = void 0;
1407
- }
1408
- });
1409
- return child;
1410
- }
1411
- handleHelperStreamError(child, streamName, error, diagnostics) {
1412
- const nodeError = error;
1413
- const helperError = this.createHelperError(`RDP helper ${streamName} stream error: ${error.message}`, nodeError.code, diagnostics);
1414
- this.rejectPendingForChild(child, helperError);
1415
- if (this.child !== child) return;
1416
- this.connected = false;
1417
- this.fatalHelperError = helperError;
1418
- if (null === child.exitCode) child.kill('SIGTERM');
1419
- this.child = void 0;
1420
- }
1421
- handleStdoutLine(child, line, diagnostics) {
1422
- if (!line.trim()) return;
1423
- let parsed;
1424
- try {
1425
- parsed = JSON.parse(line);
1426
- } catch (error) {
1427
- const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`, void 0, diagnostics);
1428
- this.rejectPendingForChild(child, protocolError);
1429
- if (this.child === child) this.shutdownHelper(protocolError);
1430
- return;
1431
- }
1432
- const pending = this.pending.get(parsed.id);
1433
- if (!pending) return void debug('dropping response for unknown request id', parsed);
1434
- this.pending.delete(parsed.id);
1435
- if (parsed.ok) return void pending.resolve(parsed.payload);
1436
- pending.reject(this.createHelperError(parsed.error.message, parsed.error.code, diagnostics));
1437
- }
1438
- captureStderrChunk(chunk, diagnostics) {
1439
- const text = chunk.toString();
1440
- if (!text.trim()) return;
1441
- debug('rdp helper stderr', {
1442
- helperPath: diagnostics.path,
1443
- pid: diagnostics.pid,
1444
- stderr: text.trim()
1445
- });
1446
- diagnostics.stderrBuffer += text;
1447
- if (diagnostics.stderrBuffer.length > MAX_STDERR_CHARS) diagnostics.stderrBuffer = diagnostics.stderrBuffer.slice(-MAX_STDERR_CHARS);
1448
- }
1449
- async send(payload) {
1450
- if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
1451
- const child = this.ensureHelperStarted();
1452
- if (null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
1453
- const id = `req-${++this.nextRequestId}`;
1454
- const diagnostics = this.helperDiagnostics.get(child);
1455
- const request = {
1456
- id,
1457
- payload
1458
- };
1459
- return new Promise((resolve, reject)=>{
1460
- this.pending.set(id, {
1461
- resolve,
1462
- reject,
1463
- child
1464
- });
1465
- child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
1466
- if (!error) return;
1467
- this.pending.delete(id);
1468
- const nodeError = error;
1469
- const timer = setTimeout(()=>{
1470
- reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`, nodeError.code, diagnostics));
1471
- }, HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS);
1472
- timer.unref?.();
1473
- });
1474
- });
1475
- }
1476
- expectOk(response, actionName) {
1477
- if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
1478
- }
1479
- rejectPending(error) {
1480
- for (const { reject } of this.pending.values())reject(error);
1481
- this.pending.clear();
1482
- }
1483
- rejectPendingForChild(child, error) {
1484
- for (const [id, pending] of this.pending)if (pending.child === child) {
1485
- pending.reject(error);
1486
- this.pending.delete(id);
1487
- }
1488
- }
1489
- createHelperError(message, code, diagnostics) {
1490
- const diagnosticParts = [
1491
- diagnostics?.path ? `path=${diagnostics.path}` : void 0,
1492
- 'number' == typeof diagnostics?.pid ? `pid=${diagnostics.pid}` : void 0,
1493
- diagnostics?.exit ? `exitCode=${diagnostics.exit.code}, signal=${diagnostics.exit.signal}` : void 0
1494
- ].filter(Boolean);
1495
- const diagnosticsSuffix = diagnosticParts.length > 0 ? `\nHelper diagnostics: ${diagnosticParts.join(', ')}` : '';
1496
- const stderrSummary = diagnostics?.stderrBuffer.trim();
1497
- const stderrSuffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
1498
- const error = new Error(`${message}${diagnosticsSuffix}${stderrSuffix}`);
1499
- if (code) error.name = code;
1500
- return error;
1501
- }
1502
- disposeReaders() {
1503
- this.stdoutReader?.close();
1504
- this.stdoutReader = void 0;
1505
- }
1506
- async shutdownHelper(rootError) {
1507
- const child = this.child;
1508
- this.child = void 0;
1509
- this.disposeReaders();
1510
- if (!child) return;
1511
- this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
1512
- if (null !== child.exitCode) return;
1513
- child.stdin.end();
1514
- const exited = Promise.race([
1515
- once(child, 'exit'),
1516
- new Promise((resolve)=>{
1517
- setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1518
- })
1519
- ]);
1520
- const result = await exited;
1521
- if ('timeout' !== result) return;
1522
- child.kill('SIGTERM');
1523
- const terminated = Promise.race([
1524
- once(child, 'exit'),
1525
- new Promise((resolve)=>{
1526
- setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1527
- })
1528
- ]);
1529
- const terminateResult = await terminated;
1530
- if ('timeout' !== terminateResult) return;
1531
- child.kill('SIGKILL');
1532
- await once(child, 'exit');
1533
- }
1534
- constructor(options){
1535
- backend_client_define_property(this, "spawnFn", void 0);
1536
- backend_client_define_property(this, "resolveHelperPath", void 0);
1537
- backend_client_define_property(this, "child", void 0);
1538
- backend_client_define_property(this, "stdoutReader", void 0);
1539
- backend_client_define_property(this, "pending", new Map());
1540
- backend_client_define_property(this, "helperDiagnostics", new WeakMap());
1541
- backend_client_define_property(this, "nextRequestId", 0);
1542
- backend_client_define_property(this, "connected", false);
1543
- backend_client_define_property(this, "fatalHelperError", void 0);
1544
- this.spawnFn = options?.spawnFn || spawn;
1545
- const overridePath = options?.helperPath;
1546
- this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
1547
- }
1548
- }
1549
- function createDefaultRDPBackendClient() {
1550
- return new HelperProcessRDPBackendClient();
1551
- }
1552
- function rdp_device_define_property(obj, key, value) {
1553
- if (key in obj) Object.defineProperty(obj, key, {
1554
- value: value,
1555
- enumerable: true,
1556
- configurable: true,
1557
- writable: true
1558
- });
1559
- else obj[key] = value;
1560
- return obj;
1561
- }
1562
- const device_debug = getDebug('rdp:device');
1563
- const device_SMOOTH_MOVE_STEPS_TAP = 8;
1564
- const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
1565
- const SMOOTH_MOVE_STEPS_DRAG = 12;
1566
- const device_SMOOTH_MOVE_DELAY_TAP = 8;
1567
- const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
1568
- const SMOOTH_MOVE_DELAY_DRAG = 10;
1569
- const device_MOUSE_MOVE_EFFECT_WAIT = 300;
1570
- const device_CLICK_HOLD_DURATION = 50;
1571
- const DRAG_HOLD_DURATION = 100;
1572
- const device_INPUT_FOCUS_DELAY = 300;
1573
- const device_INPUT_CLEAR_DELAY = 150;
1574
- const device_SCROLL_STEP_DELAY = 100;
1575
- const device_SCROLL_COMPLETE_DELAY = 500;
1576
- const DEFAULT_SCROLL_DISTANCE = 480;
1577
- const device_DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
1578
- const device_EDGE_SCROLL_STEPS = 10;
1579
- const DEFAULT_SCROLL_STEP_AMOUNT = 120;
1580
- class RDPDevice {
1581
- describe() {
1582
- const port = this.options.port || 3389;
1583
- const server = formatRdpServerAddress(this.options.host, port);
1584
- const username = this.options.username ? ` as ${this.options.username}` : '';
1585
- const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
1586
- return `RDP Device ${server}${username}${session}`;
1587
- }
1588
- async connect() {
1589
- this.throwIfDestroyed();
1590
- device_debug('connecting to rdp backend', {
1591
- host: this.options.host,
1592
- port: this.options.port,
1593
- username: this.options.username
1594
- });
1595
- const { backend: _backend, customActions: _customActions, ...config } = this.options;
1596
- this.connectionInfo = await this.backend.connect(config);
1597
- this.cursorPosition = [
1598
- Math.round(this.connectionInfo.size.width / 2),
1599
- Math.round(this.connectionInfo.size.height / 2)
1600
- ];
1601
- }
1602
- async screenshotBase64() {
1603
- this.assertConnected();
1604
- return this.backend.screenshotBase64();
1605
- }
1606
- async size() {
1607
- this.assertConnected();
1608
- return this.backend.size();
1609
- }
1610
- async destroy() {
1611
- if (this.destroyed) return;
1612
- this.destroyed = true;
1613
- this.connectionInfo = void 0;
1614
- this.cursorPosition = void 0;
1615
- await this.backend.disconnect();
1616
- }
1617
- actionSpace() {
1618
- const defaultActions = [
1619
- ...defineActionsFromInputPrimitives(this.inputPrimitives),
1620
- defineAction({
1621
- name: 'ListDisplays',
1622
- description: 'List all available displays/monitors',
1623
- call: async ()=>{
1624
- this.assertConnected();
1625
- const size = await this.size();
1626
- const server = this.connectionInfo?.server || formatRdpServerAddress(this.options.host, this.options.port || 3389);
1627
- return [
1628
- {
1629
- id: this.connectionInfo?.sessionId || this.options.host,
1630
- name: `RDP ${server} (${size.width}x${size.height})`,
1631
- primary: true
1632
- }
1633
- ];
1634
- }
1635
- })
1636
- ];
1637
- return [
1638
- ...defaultActions,
1639
- ...this.options.customActions || []
1640
- ];
1641
- }
1642
- assertConnected() {
1643
- this.throwIfDestroyed();
1644
- if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
1645
- }
1646
- throwIfDestroyed() {
1647
- if (this.destroyed) throw new Error('RDPDevice has been destroyed');
1648
- }
1649
- async moveToElement(element, options) {
1650
- this.assertConnected();
1651
- const targetX = Math.round(element.center[0]);
1652
- const targetY = Math.round(element.center[1]);
1653
- await this.movePointer(targetX, targetY, options);
1654
- }
1655
- async clearInput() {
1656
- if (this.backend.clearInput) return void await this.backend.clearInput();
1657
- await this.backend.keyPress('Control+A');
1658
- await this.backend.keyPress('Backspace');
1659
- }
1660
- edgeScrollDirection(scrollType) {
1661
- switch(scrollType){
1662
- case 'scrollToTop':
1663
- return 'up';
1664
- case 'scrollToBottom':
1665
- return 'down';
1666
- case 'scrollToLeft':
1667
- return 'left';
1668
- case 'scrollToRight':
1669
- return 'right';
1670
- case 'singleAction':
1671
- return 'down';
1672
- default:
1673
- throw new Error(`Unsupported scroll type: ${scrollType}`);
1674
- }
1675
- }
1676
- defaultScrollDistance(direction) {
1677
- const size = this.connectionInfo?.size;
1678
- if (!size) return DEFAULT_SCROLL_DISTANCE;
1679
- const isHorizontal = 'left' === direction || 'right' === direction;
1680
- const base = isHorizontal ? size.width : size.height;
1681
- return Math.max(1, Math.round(base * device_DEFAULT_SCROLL_VIEWPORT_RATIO));
1682
- }
1683
- async movePointer(targetX, targetY, options) {
1684
- this.assertConnected();
1685
- const start = this.cursorPosition || [
1686
- targetX,
1687
- targetY
1688
- ];
1689
- const steps = Math.max(1, options?.steps || 1);
1690
- const stepDelayMs = options?.stepDelayMs || 0;
1691
- for(let step = 1; step <= steps; step++){
1692
- const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
1693
- const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
1694
- await this.backend.mouseMove(x, y);
1695
- this.cursorPosition = [
1696
- x,
1697
- y
1698
- ];
1699
- if (stepDelayMs > 0 && step < steps) await sleep(stepDelayMs);
1700
- }
1701
- if (options?.settleDelayMs) await sleep(options.settleDelayMs);
1702
- }
1703
- async performWheel(direction, amount, x, y) {
1704
- let remaining = Math.abs(amount);
1705
- if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
1706
- while(remaining > 0){
1707
- const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
1708
- await this.backend.wheel(direction, chunk, x, y);
1709
- remaining -= chunk;
1710
- if (remaining > 0) await sleep(device_SCROLL_STEP_DELAY);
1711
- }
1712
- }
1713
- constructor(options){
1714
- rdp_device_define_property(this, "interfaceType", 'rdp');
1715
- rdp_device_define_property(this, "options", void 0);
1716
- rdp_device_define_property(this, "backend", void 0);
1717
- rdp_device_define_property(this, "connectionInfo", void 0);
1718
- rdp_device_define_property(this, "destroyed", false);
1719
- rdp_device_define_property(this, "cursorPosition", void 0);
1720
- rdp_device_define_property(this, "uri", void 0);
1721
- rdp_device_define_property(this, "inputPrimitives", {
1722
- pointer: {
1723
- tap: async ({ x, y })=>{
1724
- await this.movePointer(Math.round(x), Math.round(y), {
1725
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1726
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1727
- });
1728
- await this.backend.mouseButton('left', 'down');
1729
- await sleep(device_CLICK_HOLD_DURATION);
1730
- await this.backend.mouseButton('left', 'up');
1731
- },
1732
- doubleClick: async ({ x, y })=>{
1733
- await this.movePointer(Math.round(x), Math.round(y), {
1734
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1735
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1736
- });
1737
- await this.backend.mouseButton('left', 'doubleClick');
1738
- },
1739
- rightClick: async ({ x, y })=>{
1740
- await this.movePointer(Math.round(x), Math.round(y), {
1741
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1742
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1743
- });
1744
- await this.backend.mouseButton('right', 'click');
1745
- },
1746
- hover: async ({ x, y })=>{
1747
- await this.movePointer(Math.round(x), Math.round(y), {
1748
- steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1749
- stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
1750
- settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
1751
- });
1752
- },
1753
- dragAndDrop: async (from, to)=>{
1754
- await this.movePointer(Math.round(from.x), Math.round(from.y), {
1755
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1756
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1757
- });
1758
- await this.backend.mouseButton('left', 'down');
1759
- await sleep(DRAG_HOLD_DURATION);
1760
- await this.movePointer(Math.round(to.x), Math.round(to.y), {
1761
- steps: SMOOTH_MOVE_STEPS_DRAG,
1762
- stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
1763
- });
1764
- await sleep(DRAG_HOLD_DURATION);
1765
- await this.backend.mouseButton('left', 'up');
1766
- }
1767
- },
1768
- keyboard: {
1769
- typeText: async (value, opts)=>{
1770
- this.assertConnected();
1771
- const target = opts?.target;
1772
- if (target) {
1773
- await this.inputPrimitives.pointer.tap({
1774
- x: target.center[0],
1775
- y: target.center[1]
1776
- });
1777
- await sleep(device_INPUT_FOCUS_DELAY);
1778
- }
1779
- if (opts?.replace !== false) {
1780
- await this.clearInput();
1781
- await sleep(device_INPUT_CLEAR_DELAY);
1782
- }
1783
- if (opts?.focusOnly || !value) return;
1784
- await this.backend.typeText(value);
1785
- },
1786
- clearInput: async (target)=>{
1787
- this.assertConnected();
1788
- const element = target;
1789
- if (element) {
1790
- await this.inputPrimitives.pointer.tap({
1791
- x: element.center[0],
1792
- y: element.center[1]
1793
- });
1794
- await sleep(device_INPUT_FOCUS_DELAY);
1795
- }
1796
- await this.clearInput();
1797
- await sleep(device_INPUT_CLEAR_DELAY);
1798
- },
1799
- keyboardPress: async (keyName, opts)=>{
1800
- this.assertConnected();
1801
- const target = opts?.target;
1802
- if (target) await this.inputPrimitives.pointer.tap({
1803
- x: target.center[0],
1804
- y: target.center[1]
1805
- });
1806
- await this.backend.keyPress(keyName);
1807
- }
1808
- },
1809
- scroll: {
1810
- scroll: async (param)=>{
1811
- this.assertConnected();
1812
- const target = param.locate;
1813
- if (target) await this.moveToElement(target, {
1814
- steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1815
- stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
1816
- });
1817
- if (param.scrollType && 'singleAction' !== param.scrollType) {
1818
- const direction = this.edgeScrollDirection(param.scrollType);
1819
- for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
1820
- await sleep(device_SCROLL_COMPLETE_DELAY);
1821
- return;
1822
- }
1823
- await this.performWheel(param.direction || 'down', param.distance || this.defaultScrollDistance(param.direction || 'down'), target?.center[0], target?.center[1]);
1824
- await sleep(device_SCROLL_COMPLETE_DELAY);
1825
- }
1826
- }
1827
- });
1828
- const normalizedOptions = normalizeRdpConnectionConfig(options);
1829
- this.options = {
1830
- port: 3389,
1831
- securityProtocol: 'auto',
1832
- ignoreCertificate: false,
1833
- ...normalizedOptions
1834
- };
1835
- this.backend = options.backend || createDefaultRDPBackendClient();
1836
- }
1837
- }
1838
- class ComputerAgent extends Agent {
1839
- }
1840
- function createLocalComputerDevice(opts) {
1841
- return new ComputerDevice({
1842
- displayId: opts?.displayId,
1843
- customActions: opts?.customActions,
1844
- keyboardDriver: opts?.keyboardDriver,
1845
- headless: opts?.headless,
1846
- xvfbResolution: opts?.xvfbResolution
1847
- });
1848
- }
1849
- function createRDPComputerDevice(opts) {
1850
- return new RDPDevice({
1851
- host: opts.host,
1852
- port: opts.port,
1853
- username: opts.username,
1854
- password: opts.password,
1855
- domain: opts.domain,
1856
- localAddress: opts.localAddress,
1857
- adminSession: opts.adminSession,
1858
- ignoreCertificate: opts.ignoreCertificate,
1859
- securityProtocol: opts.securityProtocol,
1860
- desktopWidth: opts.desktopWidth,
1861
- desktopHeight: opts.desktopHeight,
1862
- backend: opts.backend,
1863
- customActions: opts.customActions
1864
- });
1865
- }
1866
- async function agentForComputer(opts) {
1867
- const device = createLocalComputerDevice(opts);
1868
- await device.connect();
1869
- return new ComputerAgent(device, opts);
1870
- }
1871
- const agentFromComputer = agentForComputer;
1872
- async function agentForRDPComputer(opts) {
1873
- const device = createRDPComputerDevice(opts);
1874
- await device.connect();
1875
- return new ComputerAgent(device, opts);
1876
- }
1877
- function mcp_tools_define_property(obj, key, value) {
1878
- if (key in obj) Object.defineProperty(obj, key, {
1879
- value: value,
1880
- enumerable: true,
1881
- configurable: true,
1882
- writable: true
1883
- });
1884
- else obj[key] = value;
1885
- return obj;
1886
- }
1887
- const mcp_tools_debug = getDebug('mcp:computer-tools');
1888
- const RDP_SECURITY_PROTOCOLS = [
1889
- 'auto',
1890
- 'tls',
1891
- 'nla',
1892
- 'rdp'
1893
- ];
1894
- const computerInitArgShape = {
1895
- displayId: z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
1896
- headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
1897
- host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
1898
- port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
1899
- username: z.string().optional().describe('RDP username. Requires host.'),
1900
- password: z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
1901
- domain: z.string().optional().describe('RDP domain. Requires host.'),
1902
- localAddress: z.string().optional().describe('Local source IP address for the RDP TCP connection. Requires host.'),
1903
- adminSession: z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
1904
- ignoreCertificate: z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
1905
- securityProtocol: z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
1906
- desktopWidth: z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
1907
- desktopHeight: z.number().optional().describe('Remote desktop height in pixels. Requires host.'),
1908
- ...agentBehaviorInitArgShape
1909
- };
1910
- function adaptComputerInitArgs(extracted) {
1911
- if (!extracted || 0 === Object.keys(extracted).length) return;
1912
- if (extracted.host) {
1913
- const { displayId: _d, headless: _h, ...rdpFields } = extracted;
1914
- const host = normalizeRdpHost(extracted.host);
1915
- return {
1916
- mode: 'rdp',
1917
- ...rdpFields,
1918
- host
1919
- };
1920
- }
1921
- return {
1922
- mode: 'local',
1923
- displayId: extracted.displayId,
1924
- headless: extracted.headless,
1925
- ...extractAgentBehaviorInitArgs(extracted) ?? {}
1926
- };
1927
- }
1928
- function describeConnectTarget(opts) {
1929
- if (opts?.mode === 'rdp') {
1930
- const target = opts.port ? formatRdpServerAddress(opts.host, opts.port) : formatRdpHost(opts.host);
1931
- const userSuffix = opts.username ? ` as ${opts.username}` : '';
1932
- return ` via RDP (${target}${userSuffix})`;
1933
- }
1934
- if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
1935
- return ' (Primary display)';
1936
- }
1937
- function getCliReportSessionTarget(opts) {
1938
- if (opts?.mode === 'rdp') return `rdp:${formatRdpHost(opts.host)}`;
1939
- if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
1940
- return 'primary';
1941
- }
1942
- class ComputerMidsceneTools extends BaseMidsceneTools {
1943
- getCliReportSessionName() {
1944
- return 'midscene-computer';
1945
- }
1946
- createTemporaryDevice() {
1947
- return new ComputerDevice({});
1948
- }
1949
- async ensureAgent(opts) {
1950
- const nextSignature = getAgentInitArgsSignature(opts);
1951
- if (this.agent && shouldRebuildAgentForInitArgs(this.lastInitArgsSignature, nextSignature)) {
1952
- try {
1953
- await this.agent.destroy?.();
1954
- } catch (error) {
1955
- mcp_tools_debug('Failed to destroy agent during cleanup:', error);
1956
- }
1957
- this.agent = void 0;
1958
- }
1959
- if (this.agent) return this.agent;
1960
- const reportOptions = this.readCliReportAgentOptions();
1961
- if (opts?.mode === 'rdp') {
1962
- mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
1963
- const { mode: _mode, ...rdpFields } = opts;
1964
- const agent = await agentForRDPComputer({
1965
- ...rdpFields,
1966
- ...reportOptions ?? {}
1967
- });
1968
- this.agent = agent;
1969
- this.lastInitArgsSignature = nextSignature;
1970
- return agent;
1971
- }
1972
- const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
1973
- const headless = opts?.mode === 'local' ? opts.headless : void 0;
1974
- mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
1975
- const agentOpts = {
1976
- ...displayId ? {
1977
- displayId
1978
- } : {},
1979
- ...void 0 !== headless ? {
1980
- headless
1981
- } : {},
1982
- ...extractAgentBehaviorInitArgs(opts) ?? {},
1983
- ...reportOptions ?? {}
1984
- };
1985
- const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
1986
- this.agent = agent;
1987
- this.lastInitArgsSignature = nextSignature;
1988
- return agent;
1989
- }
1990
- preparePlatformTools() {
1991
- return [
1992
- {
1993
- name: 'computer_connect',
1994
- description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/localAddress/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
1995
- schema: this.getAgentInitArgSchema(),
1996
- cli: this.getAgentInitArgCliMetadata(),
1997
- handler: async (args)=>{
1998
- const initArgs = this.extractAgentInitParam(args);
1999
- const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
2000
- this.commitCliReportSession(reportSession);
2001
- if (this.agent) {
2002
- try {
2003
- await this.agent.destroy?.();
2004
- } catch (error) {
2005
- mcp_tools_debug('Failed to destroy agent during connect:', error);
2006
- }
2007
- this.agent = void 0;
2008
- this.lastInitArgsSignature = void 0;
2009
- }
2010
- const agent = await this.ensureAgent(initArgs);
2011
- const screenshot = await agent.interface.screenshotBase64();
2012
- return {
2013
- content: [
2014
- {
2015
- type: 'text',
2016
- text: `Connected to computer${describeConnectTarget(initArgs)}`
2017
- },
2018
- ...this.buildScreenshotContent(screenshot)
2019
- ]
2020
- };
2021
- }
2022
- },
2023
- {
2024
- name: 'computer_disconnect',
2025
- description: 'Disconnect from computer and release resources',
2026
- schema: {},
2027
- handler: this.createDisconnectHandler('computer')
2028
- },
2029
- {
2030
- name: 'computer_list_displays',
2031
- description: 'List all available displays/monitors',
2032
- schema: {},
2033
- handler: async ()=>{
2034
- const displays = await ComputerDevice.listDisplays();
2035
- return {
2036
- content: [
2037
- {
2038
- type: 'text',
2039
- text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
2040
- }
2041
- ]
2042
- };
2043
- }
2044
- }
2045
- ];
2046
- }
2047
- constructor(...args){
2048
- super(...args), mcp_tools_define_property(this, "lastInitArgsSignature", void 0), mcp_tools_define_property(this, "initArgSpec", {
2049
- namespace: 'computer',
2050
- shape: computerInitArgShape,
2051
- cli: {
2052
- preferBareKeys: true
2053
- },
2054
- adapt: (extracted)=>adaptComputerInitArgs(extracted)
2055
- });
2056
- }
2057
- }
2058
- class ComputerMCPServer extends BaseMCPServer {
2059
- createToolsManager() {
2060
- return new ComputerMidsceneTools();
2061
- }
2062
- constructor(toolsManager){
2063
- super({
2064
- name: '@midscene/computer-mcp',
2065
- version: "1.9.8-beta-20260618091332.0",
2066
- description: 'Control the computer desktop using natural language commands'
2067
- }, toolsManager);
2068
- }
2069
- }
2070
- function mcpServerForAgent(agent) {
2071
- return createMCPServerLauncher({
2072
- agent,
2073
- platformName: 'Computer',
2074
- ToolsManagerClass: ComputerMidsceneTools,
2075
- MCPServerClass: ComputerMCPServer
2076
- });
2077
- }
2078
- async function mcpKitForAgent(agent) {
2079
- const toolsManager = new ComputerMidsceneTools();
2080
- const computerAgent = agent instanceof ComputerAgent ? agent : agent;
2081
- toolsManager.setAgent(computerAgent);
2082
- await toolsManager.initTools();
2083
- return {
2084
- description: 'Midscene MCP Kit for computer desktop automation',
2085
- tools: toolsManager.getToolDefinitions()
2086
- };
2087
- }
2088
- export { ComputerMCPServer, mcpKitForAgent, mcpServerForAgent };