@midscene/computer 1.9.8 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2141 +0,0 @@
1
- "use strict";
2
- const __rslib_import_meta_url__ = /*#__PURE__*/ function() {
3
- return 'undefined' == typeof document ? new (require('url'.replace('', ''))).URL('file:' + __filename).href : document.currentScript && document.currentScript.src || new URL('main.js', document.baseURI).href;
4
- }();
5
- var __webpack_require__ = {};
6
- (()=>{
7
- __webpack_require__.n = (module)=>{
8
- var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
9
- __webpack_require__.d(getter, {
10
- a: getter
11
- });
12
- return getter;
13
- };
14
- })();
15
- (()=>{
16
- __webpack_require__.d = (exports1, definition)=>{
17
- for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
18
- enumerable: true,
19
- get: definition[key]
20
- });
21
- };
22
- })();
23
- (()=>{
24
- __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
25
- })();
26
- (()=>{
27
- __webpack_require__.r = (exports1)=>{
28
- if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
29
- value: 'Module'
30
- });
31
- Object.defineProperty(exports1, '__esModule', {
32
- value: true
33
- });
34
- };
35
- })();
36
- var __webpack_exports__ = {};
37
- __webpack_require__.r(__webpack_exports__);
38
- __webpack_require__.d(__webpack_exports__, {
39
- mcpKitForAgent: ()=>mcpKitForAgent,
40
- mcpServerForAgent: ()=>mcpServerForAgent,
41
- ComputerMCPServer: ()=>ComputerMCPServer
42
- });
43
- const mcp_namespaceObject = require("@midscene/shared/mcp");
44
- const agent_namespaceObject = require("@midscene/core/agent");
45
- const external_node_child_process_namespaceObject = require("node:child_process");
46
- const external_node_fs_namespaceObject = require("node:fs");
47
- const external_node_module_namespaceObject = require("node:module");
48
- const external_node_path_namespaceObject = require("node:path");
49
- const external_node_url_namespaceObject = require("node:url");
50
- const device_namespaceObject = require("@midscene/core/device");
51
- const utils_namespaceObject = require("@midscene/core/utils");
52
- const img_namespaceObject = require("@midscene/shared/img");
53
- const logger_namespaceObject = require("@midscene/shared/logger");
54
- const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
55
- var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
56
- const external_node_assert_namespaceObject = require("node:assert");
57
- var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
58
- function _define_property(obj, key, value) {
59
- if (key in obj) Object.defineProperty(obj, key, {
60
- value: value,
61
- enumerable: true,
62
- configurable: true,
63
- writable: true
64
- });
65
- else obj[key] = value;
66
- return obj;
67
- }
68
- class ComputerInputDriver {
69
- destroy() {
70
- if (this.destroyed) return;
71
- this.destroyed = true;
72
- this.rejectPendingInputDelays();
73
- }
74
- getScreenSize() {
75
- return this.getLibnutOrThrow('getScreenSize').getScreenSize();
76
- }
77
- getMousePos() {
78
- return this.getLibnutOrThrow('getMousePos').getMousePos();
79
- }
80
- moveMouse(x, y) {
81
- this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
82
- }
83
- focusActiveWindow() {
84
- const lib = this.getLibnutOrThrow('focusActiveWindow');
85
- if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
86
- try {
87
- const handle = lib.getActiveWindow();
88
- if (!handle) return false;
89
- lib.focusWindow(handle);
90
- return true;
91
- } catch (error) {
92
- this.options.debug(`focusActiveWindow failed: ${error}`);
93
- return false;
94
- }
95
- }
96
- getActiveWindowRect() {
97
- const lib = this.getLibnutOrThrow('getActiveWindowRect');
98
- if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
99
- try {
100
- const handle = lib.getActiveWindow();
101
- if (!handle) return null;
102
- const rect = lib.getWindowRect(handle);
103
- if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
104
- return rect;
105
- } catch (error) {
106
- this.options.debug(`getActiveWindowRect failed: ${error}`);
107
- return null;
108
- }
109
- }
110
- mouseClick(button, double) {
111
- const lib = this.getLibnutOrThrow('mouseClick');
112
- if (void 0 !== double) lib.mouseClick(button, double);
113
- else if (void 0 !== button) lib.mouseClick(button);
114
- else lib.mouseClick();
115
- }
116
- mouseToggle(state, button = 'left') {
117
- this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
118
- }
119
- scrollMouse(x, y) {
120
- this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
121
- }
122
- async emitScrollDetents(dx, dy, detents, delayMs) {
123
- this.assertActive('emitScrollDetents');
124
- for(let i = 0; i < detents; i++){
125
- this.scrollMouse(dx, dy);
126
- if (i < detents - 1) await this.delay(delayMs);
127
- }
128
- }
129
- keyTap(key, modifiers) {
130
- const lib = this.getLibnutOrThrow('keyTap');
131
- if (void 0 !== modifiers) lib.keyTap(key, modifiers);
132
- else lib.keyTap(key);
133
- }
134
- sendKeyViaAppleScript(key, modifiers = []) {
135
- this.assertActive('sendKeyViaAppleScript');
136
- this.options.sendKeyViaAppleScript(key, modifiers);
137
- }
138
- sendKey(key, modifiers = []) {
139
- if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
140
- if (modifiers.length > 0) this.keyTap(key, modifiers);
141
- else this.keyTap(key);
142
- }
143
- runPhasedScroll(direction, pixels, steps) {
144
- this.assertActive('runPhasedScroll');
145
- return this.options.runPhasedScroll(direction, pixels, steps);
146
- }
147
- async delay(ms) {
148
- this.assertActive('delay');
149
- return new Promise((resolve, reject)=>{
150
- const waitRef = {
151
- timeoutId: setTimeout(()=>{
152
- this.pendingInputDelayWaits.delete(waitRef);
153
- try {
154
- this.assertActive('delay');
155
- resolve();
156
- } catch (error) {
157
- reject(error);
158
- }
159
- }, ms),
160
- reject
161
- };
162
- this.pendingInputDelayWaits.add(waitRef);
163
- });
164
- }
165
- async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
166
- const currentPos = this.getMousePos();
167
- for(let i = 1; i <= steps; i++){
168
- const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
169
- const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
170
- this.moveMouse(stepX, stepY);
171
- await this.delay(stepDelay);
172
- }
173
- }
174
- async withMouseButton(button, run) {
175
- this.mouseToggle('down', button);
176
- try {
177
- return await run();
178
- } finally{
179
- this.releaseMouseButton(button);
180
- }
181
- }
182
- getLibnutOrThrow(methodName) {
183
- this.assertActive(methodName);
184
- const libnut = this.options.getLibnut();
185
- external_node_assert_default()(libnut, 'libnut not initialized');
186
- return libnut;
187
- }
188
- assertActive(methodName) {
189
- if (this.destroyed) throw this.createDestroyedError(methodName);
190
- }
191
- createDestroyedError(methodName) {
192
- return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
193
- }
194
- releaseMouseButton(button) {
195
- try {
196
- const libnut = this.options.getLibnut();
197
- external_node_assert_default()(libnut, 'libnut not initialized');
198
- libnut.mouseToggle('up', button);
199
- } catch (error) {
200
- this.options.debug(`Failed to release mouse button ${button}: ${error}`);
201
- }
202
- }
203
- rejectPendingInputDelays() {
204
- const error = this.createDestroyedError('in-flight input');
205
- for (const waitRef of this.pendingInputDelayWaits){
206
- clearTimeout(waitRef.timeoutId);
207
- waitRef.reject(error);
208
- }
209
- this.pendingInputDelayWaits.clear();
210
- }
211
- constructor(options){
212
- _define_property(this, "options", void 0);
213
- _define_property(this, "destroyed", void 0);
214
- _define_property(this, "pendingInputDelayWaits", void 0);
215
- this.options = options;
216
- this.destroyed = false;
217
- this.pendingInputDelayWaits = new Set();
218
- }
219
- }
220
- const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
221
- function checkXvfbInstalled() {
222
- try {
223
- (0, external_node_child_process_namespaceObject.execSync)('which Xvfb', {
224
- stdio: 'ignore'
225
- });
226
- return true;
227
- } catch {
228
- return false;
229
- }
230
- }
231
- function findAvailableDisplay(startFrom = 99) {
232
- for(let n = startFrom; n < startFrom + 100; n++)if (!(0, external_node_fs_namespaceObject.existsSync)(`/tmp/.X${n}-lock`)) return n;
233
- throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
234
- }
235
- function startXvfb(options) {
236
- const resolution = options?.resolution || '1920x1080x24';
237
- const displayNum = options?.displayNumber ?? findAvailableDisplay();
238
- const display = `:${displayNum}`;
239
- return new Promise((resolve, reject)=>{
240
- debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
241
- const xvfbProcess = (0, external_node_child_process_namespaceObject.spawn)('Xvfb', [
242
- display,
243
- '-screen',
244
- '0',
245
- resolution,
246
- '-ac',
247
- '-nolisten',
248
- 'tcp'
249
- ], {
250
- stdio: 'ignore'
251
- });
252
- let settled = false;
253
- xvfbProcess.on('error', (err)=>{
254
- if (!settled) {
255
- settled = true;
256
- reject(new Error(`Failed to start Xvfb: ${err.message}`));
257
- }
258
- });
259
- xvfbProcess.on('exit', (code)=>{
260
- if (!settled) {
261
- settled = true;
262
- reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
263
- }
264
- });
265
- const instance = {
266
- process: xvfbProcess,
267
- display,
268
- stop () {
269
- try {
270
- xvfbProcess.kill('SIGTERM');
271
- } catch {}
272
- }
273
- };
274
- setTimeout(()=>{
275
- if (!settled) {
276
- settled = true;
277
- debugXvfb(`Xvfb started on display ${display}`);
278
- resolve(instance);
279
- }
280
- }, 500);
281
- });
282
- }
283
- function needsXvfb(explicitOpt) {
284
- if ('linux' !== process.platform) return false;
285
- return true === explicitOpt;
286
- }
287
- function device_define_property(obj, key, value) {
288
- if (key in obj) Object.defineProperty(obj, key, {
289
- value: value,
290
- enumerable: true,
291
- configurable: true,
292
- writable: true
293
- });
294
- else obj[key] = value;
295
- return obj;
296
- }
297
- const SMOOTH_MOVE_STEPS_TAP = 8;
298
- const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
299
- const SMOOTH_MOVE_DELAY_TAP = 8;
300
- const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
301
- const MOUSE_MOVE_EFFECT_WAIT = 300;
302
- const CLICK_SETTLE_DELAY = 50;
303
- const CLICK_HOLD_DURATION = 100;
304
- const CLICK_FOCUS_SETTLE_DELAY = 120;
305
- const INPUT_FOCUS_DELAY = 300;
306
- const INPUT_CLEAR_DELAY = 150;
307
- const SCROLL_STEP_DELAY = 100;
308
- const SCROLL_COMPLETE_DELAY = 500;
309
- const EDGE_SCROLL_TOTAL_PX = 50000;
310
- const EDGE_SCROLL_STEPS = 400;
311
- const PHASED_PIXELS_PER_STEP = 30;
312
- const PHASED_MIN_STEPS = 10;
313
- const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
314
- const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
315
- const LIBNUT_FALLBACK_MAX_DETENTS = 200;
316
- const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
317
- const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
318
- const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
319
- const EDGE_SCROLL_SPEC = {
320
- scrollToTop: {
321
- direction: 'up',
322
- key: 'home',
323
- libnut: [
324
- 0,
325
- 1
326
- ]
327
- },
328
- scrollToBottom: {
329
- direction: 'down',
330
- key: 'end',
331
- libnut: [
332
- 0,
333
- -1
334
- ]
335
- },
336
- scrollToLeft: {
337
- direction: 'left',
338
- key: 'home',
339
- libnut: [
340
- -1,
341
- 0
342
- ]
343
- },
344
- scrollToRight: {
345
- direction: 'right',
346
- key: 'end',
347
- libnut: [
348
- 1,
349
- 0
350
- ]
351
- }
352
- };
353
- const APPLESCRIPT_KEY_CODE_MAP = {
354
- return: 36,
355
- enter: 36,
356
- tab: 48,
357
- space: 49,
358
- backspace: 51,
359
- delete: 51,
360
- escape: 53,
361
- forwarddelete: 117,
362
- left: 123,
363
- right: 124,
364
- down: 125,
365
- up: 126,
366
- home: 115,
367
- end: 119,
368
- pageup: 116,
369
- pagedown: 121,
370
- f1: 122,
371
- f2: 120,
372
- f3: 99,
373
- f4: 118,
374
- f5: 96,
375
- f6: 97,
376
- f7: 98,
377
- f8: 100,
378
- f9: 101,
379
- f10: 109,
380
- f11: 103,
381
- f12: 111
382
- };
383
- const APPLESCRIPT_MODIFIER_MAP = {
384
- command: 'command down',
385
- cmd: 'command down',
386
- control: 'control down',
387
- ctrl: 'control down',
388
- shift: 'shift down',
389
- alt: 'option down',
390
- option: 'option down',
391
- meta: 'command down'
392
- };
393
- function sendKeyViaAppleScript(key, modifiers = []) {
394
- const lowerKey = key.toLowerCase();
395
- const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
396
- const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
397
- const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
398
- let script;
399
- if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
400
- else {
401
- const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
402
- script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
403
- }
404
- debugDevice('sendKeyViaAppleScript', {
405
- key,
406
- modifiers,
407
- script
408
- });
409
- (0, external_node_child_process_namespaceObject.execFileSync)("osascript", [
410
- '-e',
411
- script
412
- ]);
413
- }
414
- let device_libnut = null;
415
- let libnutLoadError = null;
416
- async function getLibnut() {
417
- if (device_libnut) return device_libnut;
418
- if (libnutLoadError) throw libnutLoadError;
419
- try {
420
- const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
421
- const libnutModule = require1('@computer-use/libnut/dist/import_libnut');
422
- device_libnut = libnutModule.libnut;
423
- if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
424
- return device_libnut;
425
- } catch (error) {
426
- libnutLoadError = error;
427
- throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
428
- }
429
- }
430
- const debugDevice = (0, logger_namespaceObject.getDebug)('computer:device');
431
- const warnDevice = (0, logger_namespaceObject.getDebug)('computer:device', {
432
- console: true
433
- });
434
- const debugComputerInput = (0, logger_namespaceObject.getDebug)('computer:input', {
435
- console: true
436
- });
437
- const WINDOWS_UIPI_DOC_URL = 'https://midscenejs.com/computer-getting-started#windows-clicks-have-no-effect-on-some-apps';
438
- function resolvePackageRoot(helperName) {
439
- const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
440
- let pkgRoot = null;
441
- try {
442
- pkgRoot = (0, external_node_path_namespaceObject.dirname)(require1.resolve('@midscene/computer/package.json'));
443
- } catch {
444
- const hereDir = (0, external_node_path_namespaceObject.dirname)((0, external_node_url_namespaceObject.fileURLToPath)(__rslib_import_meta_url__));
445
- for (const candidate of [
446
- (0, external_node_path_namespaceObject.resolve)(hereDir, '..'),
447
- (0, external_node_path_namespaceObject.resolve)(hereDir, '../..')
448
- ])if ((0, external_node_fs_namespaceObject.existsSync)((0, external_node_path_namespaceObject.resolve)(candidate, 'package.json'))) {
449
- pkgRoot = candidate;
450
- break;
451
- }
452
- }
453
- if (!pkgRoot) {
454
- debugDevice(`${helperName}: cannot locate @midscene/computer package root`);
455
- return null;
456
- }
457
- return pkgRoot;
458
- }
459
- let phasedScrollBinaryPath;
460
- function getPhasedScrollBinary() {
461
- if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
462
- if ('darwin' !== process.platform) {
463
- phasedScrollBinaryPath = null;
464
- return null;
465
- }
466
- const pkgRoot = resolvePackageRoot('phased-scroll');
467
- if (!pkgRoot) {
468
- phasedScrollBinaryPath = null;
469
- return null;
470
- }
471
- const binPath = (0, external_node_path_namespaceObject.resolve)(pkgRoot, 'bin/darwin/phased-scroll');
472
- if (!(0, external_node_fs_namespaceObject.existsSync)(binPath)) {
473
- debugDevice('phased-scroll binary not found at', binPath);
474
- phasedScrollBinaryPath = null;
475
- return null;
476
- }
477
- try {
478
- const st = (0, external_node_fs_namespaceObject.statSync)(binPath);
479
- if ((73 & st.mode) === 0) {
480
- (0, external_node_fs_namespaceObject.chmodSync)(binPath, 493);
481
- debugDevice('phased-scroll: restored executable bit on', binPath);
482
- }
483
- } catch (err) {
484
- debugDevice('phased-scroll: chmod self-heal failed', err);
485
- }
486
- phasedScrollBinaryPath = binPath;
487
- return binPath;
488
- }
489
- let displayInfoBinaryPath;
490
- function getDisplayInfoBinary() {
491
- if (void 0 !== displayInfoBinaryPath) return displayInfoBinaryPath;
492
- if ('darwin' !== process.platform) {
493
- displayInfoBinaryPath = null;
494
- return null;
495
- }
496
- const pkgRoot = resolvePackageRoot('display-info');
497
- if (!pkgRoot) {
498
- displayInfoBinaryPath = null;
499
- return null;
500
- }
501
- const binPath = (0, external_node_path_namespaceObject.resolve)(pkgRoot, 'bin/darwin/display-info');
502
- if (!(0, external_node_fs_namespaceObject.existsSync)(binPath)) {
503
- debugDevice('display-info binary not found at', binPath);
504
- displayInfoBinaryPath = null;
505
- return null;
506
- }
507
- displayInfoBinaryPath = binPath;
508
- return binPath;
509
- }
510
- function isFiniteNumber(value) {
511
- return 'number' == typeof value && Number.isFinite(value);
512
- }
513
- function isDarwinDisplayGeometry(value) {
514
- if (!value || 'object' != typeof value) return false;
515
- const candidate = value;
516
- return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
517
- }
518
- function readDarwinDisplayGeometries() {
519
- const bin = getDisplayInfoBinary();
520
- if (!bin) return [];
521
- try {
522
- const output = (0, external_node_child_process_namespaceObject.execFileSync)(bin, [], {
523
- encoding: 'utf8',
524
- stdio: [
525
- 'ignore',
526
- 'pipe',
527
- 'pipe'
528
- ]
529
- });
530
- const parsed = JSON.parse(output);
531
- return Array.isArray(parsed.displays) ? parsed.displays.filter(isDarwinDisplayGeometry) : [];
532
- } catch (error) {
533
- debugDevice('display-info helper failed:', error);
534
- return [];
535
- }
536
- }
537
- function readDarwinFrontmostApplication() {
538
- try {
539
- const output = (0, external_node_child_process_namespaceObject.execFileSync)("osascript", [
540
- '-e',
541
- 'tell application "System Events"\nset frontApp to first application process whose frontmost is true\nreturn (unix id of frontApp as string) & " " & (name of frontApp as string)\nend tell'
542
- ]).toString().trim();
543
- const [pidText, ...nameParts] = output.split('\t');
544
- const pid = Number(pidText);
545
- if (!Number.isInteger(pid) || pid <= 0) return;
546
- return {
547
- pid,
548
- name: nameParts.join('\t')
549
- };
550
- } catch (error) {
551
- debugDevice('Failed to read macOS frontmost application:', error);
552
- return;
553
- }
554
- }
555
- async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
556
- await inputDriver.delay(CLICK_SETTLE_DELAY);
557
- const current = inputDriver.getMousePos();
558
- debugComputerInput('tap mouse moved %o', {
559
- reason,
560
- target: {
561
- x: targetX,
562
- y: targetY
563
- },
564
- current,
565
- drift: {
566
- x: current.x - targetX,
567
- y: current.y - targetY
568
- }
569
- });
570
- await inputDriver.withMouseButton('left', async ()=>{
571
- debugComputerInput('tap mouse down %o', {
572
- reason
573
- });
574
- await inputDriver.delay(holdDuration);
575
- });
576
- debugComputerInput('tap mouse up %o', {
577
- reason
578
- });
579
- }
580
- function resolveDarwinDisplayGeometryFromList(displayId, displays) {
581
- if (!displays.length) return;
582
- const screenIndex = void 0 === displayId || '' === displayId ? 0 : Number(displayId);
583
- if (!Number.isInteger(screenIndex) || screenIndex < 0) return void debugDevice('Invalid macOS display id for display geometry:', displayId);
584
- if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
585
- return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
586
- }
587
- function resolveDisplayGeometry(displayId) {
588
- if ('darwin' !== process.platform) return;
589
- return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
590
- }
591
- function mapDisplayLocalPointToGlobal(point, geometry) {
592
- if (!geometry) return point;
593
- return {
594
- x: point.x + geometry.bounds.x,
595
- y: point.y + geometry.bounds.y
596
- };
597
- }
598
- let phasedScrollExecWarned = false;
599
- function runPhasedScroll(direction, pixels, steps) {
600
- const bin = getPhasedScrollBinary();
601
- if (!bin) return false;
602
- try {
603
- const res = (0, external_node_child_process_namespaceObject.spawnSync)(bin, [
604
- direction,
605
- String(Math.max(1, Math.round(pixels))),
606
- String(steps)
607
- ], {
608
- stdio: 'ignore'
609
- });
610
- if (0 === res.status) return true;
611
- if (!phasedScrollExecWarned) {
612
- phasedScrollExecWarned = true;
613
- const hint = null === res.status ? `signal ${res.signal ?? 'unknown'}; the binary may not be executable (npm tarball extraction can drop the +x bit) or may be blocked by quarantine. Try: chmod +x "${bin}"` : 'this usually means Accessibility permission has not been granted to the host process (System Settings → Privacy & Security → Accessibility)';
614
- console.warn(`[@midscene/computer] phased-scroll helper failed (exit=${res.status}, signal=${res.signal ?? 'none'}); falling back to keyboard/libnut. ${hint}`);
615
- }
616
- debugDevice('phased-scroll exited non-zero', res.status, res.signal, res.error);
617
- return false;
618
- } catch (err) {
619
- if (!phasedScrollExecWarned) {
620
- phasedScrollExecWarned = true;
621
- console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
622
- }
623
- debugDevice('phased-scroll spawn failed', err);
624
- return false;
625
- }
626
- }
627
- const KEY_NAME_MAP = {
628
- windows: 'win',
629
- win: 'win',
630
- ctrl: 'control',
631
- esc: 'escape',
632
- del: 'delete',
633
- ins: 'insert',
634
- pgup: 'pageup',
635
- pgdn: 'pagedown',
636
- arrowup: 'up',
637
- arrowdown: 'down',
638
- arrowleft: 'left',
639
- arrowright: 'right',
640
- volumedown: 'audio_vol_down',
641
- volumeup: 'audio_vol_up',
642
- mediavolumedown: 'audio_vol_down',
643
- mediavolumeup: 'audio_vol_up',
644
- mute: 'audio_mute',
645
- mediamute: 'audio_mute',
646
- mediaplay: 'audio_play',
647
- mediapause: 'audio_pause',
648
- mediaplaypause: 'audio_play',
649
- mediastop: 'audio_stop',
650
- medianexttrack: 'audio_next',
651
- mediaprevioustrack: 'audio_prev',
652
- medianext: 'audio_next',
653
- mediaprev: 'audio_prev'
654
- };
655
- const PRIMARY_KEY_MAP = {
656
- command: 'cmd',
657
- cmd: 'cmd',
658
- meta: 'meta',
659
- control: 'control',
660
- ctrl: 'control',
661
- shift: 'shift',
662
- alt: 'alt',
663
- option: 'alt'
664
- };
665
- function normalizeKeyName(key) {
666
- const lowerKey = key.toLowerCase();
667
- return KEY_NAME_MAP[lowerKey] || lowerKey;
668
- }
669
- function normalizePrimaryKey(key) {
670
- const lowerKey = key.toLowerCase();
671
- if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
672
- return KEY_NAME_MAP[lowerKey] || lowerKey;
673
- }
674
- class ComputerDevice {
675
- describe() {
676
- return this.description || 'Computer Device';
677
- }
678
- static async listDisplays() {
679
- try {
680
- const displays = await external_screenshot_desktop_default().listDisplays();
681
- return displays.map((d)=>({
682
- id: String(d.id),
683
- name: d.name || `Display ${d.id}`,
684
- primary: d.primary || false
685
- }));
686
- } catch (error) {
687
- debugDevice(`Failed to list displays: ${error}`);
688
- return [];
689
- }
690
- }
691
- async connect() {
692
- debugDevice('Connecting to computer device');
693
- try {
694
- const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
695
- if (needsXvfb(headless)) {
696
- if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
697
- this.xvfbInstance = await startXvfb({
698
- resolution: this.options?.xvfbResolution
699
- });
700
- process.env.DISPLAY = this.xvfbInstance.display;
701
- debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
702
- this.xvfbCleanup = ()=>{
703
- if (this.xvfbInstance) {
704
- this.xvfbInstance.stop();
705
- this.xvfbInstance = void 0;
706
- }
707
- };
708
- process.on('exit', this.xvfbCleanup);
709
- process.on('SIGINT', this.xvfbCleanup);
710
- process.on('SIGTERM', this.xvfbCleanup);
711
- }
712
- device_libnut = await getLibnut();
713
- this.displayGeometry = resolveDisplayGeometry(this.displayId);
714
- const size = await this.size();
715
- const displays = await ComputerDevice.listDisplays();
716
- const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
717
- this.description = `
718
- Type: Computer
719
- Platform: ${process.platform}
720
- Display: ${this.displayId || 'Primary'}
721
- Screen Size: ${size.width}x${size.height}
722
- Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
723
- `;
724
- debugDevice('Computer device connected', this.description);
725
- await this.healthCheck();
726
- } catch (error) {
727
- if (this.xvfbInstance) {
728
- this.xvfbInstance.stop();
729
- this.xvfbInstance = void 0;
730
- }
731
- debugDevice(`Failed to connect: ${error}`);
732
- throw new Error(`Unable to connect to computer device: ${error}`);
733
- }
734
- }
735
- async healthCheck() {
736
- console.log('[HealthCheck] Starting health check...');
737
- console.log("[HealthCheck] @midscene/computer v1.9.8");
738
- console.log('[HealthCheck] Taking screenshot...');
739
- const screenshotTimeout = 15000;
740
- let timeoutId;
741
- const timeoutPromise = new Promise((_, reject)=>{
742
- timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
743
- });
744
- const base64 = await Promise.race([
745
- this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
746
- timeoutPromise
747
- ]);
748
- console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
749
- console.log('[HealthCheck] Moving mouse...');
750
- const startPos = this.inputDriver.getMousePos();
751
- console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
752
- const offsetX = Math.floor(40 * Math.random()) + 10;
753
- const offsetY = Math.floor(40 * Math.random()) + 10;
754
- const targetX = startPos.x + offsetX;
755
- const targetY = startPos.y + offsetY;
756
- console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
757
- this.inputDriver.moveMouse(targetX, targetY);
758
- await (0, utils_namespaceObject.sleep)(50);
759
- const movedPos = this.inputDriver.getMousePos();
760
- console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
761
- const deltaX = Math.abs(movedPos.x - targetX);
762
- const deltaY = Math.abs(movedPos.y - targetY);
763
- if (deltaX > 5 || deltaY > 5) {
764
- const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
765
- warnDevice(msg);
766
- }
767
- if ('win32' === process.platform && !this.isRunningAsAdmin()) {
768
- const hint = [
769
- 'Heads-up: Midscene is not running as Administrator.',
770
- 'If clicks or key presses have no effect while the cursor still moves to the right position,',
771
- `see the Windows permission troubleshooting guide: ${WINDOWS_UIPI_DOC_URL}`
772
- ].join(' ');
773
- warnDevice(`[HealthCheck] ${hint}`);
774
- }
775
- this.inputDriver.moveMouse(startPos.x, startPos.y);
776
- console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
777
- console.log('[HealthCheck] Listing monitors...');
778
- const displays = await ComputerDevice.listDisplays();
779
- if (displays.length > 0) {
780
- console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
781
- for (const display of displays){
782
- const primaryTag = display.primary ? ' (primary)' : '';
783
- console.log(`[HealthCheck] - id=${display.id}, name=${display.name}${primaryTag}`);
784
- }
785
- } else console.log('[HealthCheck] No monitors detected');
786
- console.log('[HealthCheck] Health check passed');
787
- }
788
- isRunningAsAdmin() {
789
- if ('win32' !== process.platform) return false;
790
- if (void 0 !== this.adminCheckCache) return this.adminCheckCache;
791
- try {
792
- (0, external_node_child_process_namespaceObject.execSync)('net session', {
793
- stdio: 'pipe'
794
- });
795
- this.adminCheckCache = true;
796
- } catch {
797
- this.adminCheckCache = false;
798
- }
799
- return this.adminCheckCache;
800
- }
801
- async screenshotBase64() {
802
- if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
803
- debugDevice('Taking screenshot', {
804
- displayId: this.displayId
805
- });
806
- const options = {
807
- format: 'png'
808
- };
809
- if (void 0 !== this.displayId) if ('darwin' === process.platform) {
810
- const screenIndex = Number(this.displayId);
811
- if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
812
- } else options.screen = this.displayId;
813
- debugDevice('Screenshot options', options);
814
- const MAX_ATTEMPTS = 3;
815
- const RETRY_DELAY_MS = 300;
816
- let lastRawMessage = '';
817
- for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
818
- const buffer = await external_screenshot_desktop_default()(options);
819
- if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
820
- return (0, img_namespaceObject.createImgBase64ByFormat)('png', buffer.toString('base64'));
821
- } catch (error) {
822
- lastRawMessage = error instanceof Error ? error.message : String(error);
823
- const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
824
- const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
825
- debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
826
- if (!willRetry) break;
827
- await (0, utils_namespaceObject.sleep)(RETRY_DELAY_MS);
828
- }
829
- if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
830
-
831
- Please follow these steps:
832
- 1. Open System Settings > Privacy & Security > Screen Recording
833
- 2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
834
- 3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
835
-
836
- Original error: ${lastRawMessage}`);
837
- throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
838
- }
839
- async size() {
840
- if (this.displayGeometry) return {
841
- width: Math.round(this.displayGeometry.bounds.width),
842
- height: Math.round(this.displayGeometry.bounds.height)
843
- };
844
- try {
845
- const screenSize = this.inputDriver.getScreenSize();
846
- return {
847
- width: screenSize.width,
848
- height: screenSize.height
849
- };
850
- } catch (error) {
851
- debugDevice(`Failed to get screen size: ${error}`);
852
- throw new Error(`Failed to get screen size: ${error}`);
853
- }
854
- }
855
- toGlobalPoint(point) {
856
- return mapDisplayLocalPointToGlobal(point, this.displayGeometry);
857
- }
858
- async typeViaClipboard(text) {
859
- debugDevice('Using clipboard to input text', {
860
- textLength: text.length,
861
- preview: text.substring(0, 20)
862
- });
863
- const clipboardy = await import("clipboardy");
864
- const oldClipboard = await clipboardy.default.read().catch(()=>'');
865
- try {
866
- await clipboardy.default.write(text);
867
- await this.inputDriver.delay(50);
868
- if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
869
- 'command'
870
- ]);
871
- else {
872
- const modifier = 'darwin' === process.platform ? 'command' : 'control';
873
- this.inputDriver.keyTap('v', [
874
- modifier
875
- ]);
876
- }
877
- await this.inputDriver.delay(100);
878
- } finally{
879
- if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
880
- debugDevice('Failed to restore clipboard content');
881
- });
882
- }
883
- }
884
- async smartTypeString(text) {
885
- await this.typeViaClipboard(text);
886
- }
887
- async selectAllAndDelete() {
888
- if (this.useAppleScript) {
889
- this.inputDriver.sendKeyViaAppleScript('a', [
890
- 'command'
891
- ]);
892
- await this.inputDriver.delay(50);
893
- this.inputDriver.sendKeyViaAppleScript('backspace', []);
894
- return;
895
- }
896
- const modifier = 'darwin' === process.platform ? 'command' : 'control';
897
- this.inputDriver.keyTap('a', [
898
- modifier
899
- ]);
900
- await this.inputDriver.delay(50);
901
- this.inputDriver.keyTap('backspace');
902
- }
903
- async pressKeyboardShortcut(keyName) {
904
- const keys = keyName.split('+');
905
- const modifiers = keys.slice(0, -1).map(normalizeKeyName);
906
- const key = normalizePrimaryKey(keys[keys.length - 1]);
907
- debugDevice('KeyboardPress', {
908
- original: keyName,
909
- key,
910
- modifiers,
911
- driver: this.useAppleScript ? "applescript" : 'libnut'
912
- });
913
- this.inputDriver.sendKey(key, modifiers);
914
- }
915
- resolveUntargetedScrollPoint(screenSize) {
916
- if ('win32' === process.platform) {
917
- const activeWindowRect = this.inputDriver.getActiveWindowRect();
918
- if (activeWindowRect) return {
919
- x: activeWindowRect.x + activeWindowRect.width / 2,
920
- y: activeWindowRect.y + activeWindowRect.height / 2
921
- };
922
- }
923
- return this.toGlobalPoint({
924
- x: screenSize.width / 2,
925
- y: screenSize.height / 2
926
- });
927
- }
928
- async moveMouseToScrollTarget(param) {
929
- if (param.locate) {
930
- const element = param.locate;
931
- const [x, y] = element.center;
932
- const point = this.toGlobalPoint({
933
- x,
934
- y
935
- });
936
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
937
- return;
938
- }
939
- const screenSize = await this.size();
940
- if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
941
- const point = this.resolveUntargetedScrollPoint(screenSize);
942
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
943
- await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
944
- return screenSize;
945
- }
946
- async performScroll(param) {
947
- let screenSize = await this.moveMouseToScrollTarget(param);
948
- const scrollType = param?.scrollType;
949
- const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
950
- if (edgeSpec) {
951
- if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
952
- if (this.useAppleScript) {
953
- this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
954
- await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
955
- return;
956
- }
957
- const [ux, uy] = edgeSpec.libnut;
958
- await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
959
- return;
960
- }
961
- if ('singleAction' === scrollType || !scrollType) {
962
- const direction = param?.direction || 'down';
963
- const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
964
- const isHorizontal = 'left' === direction || 'right' === direction;
965
- let distance = param?.distance ?? void 0;
966
- if (!distance) {
967
- screenSize ??= await this.size();
968
- const base = isHorizontal ? screenSize.width : screenSize.height;
969
- distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
970
- }
971
- if (isKnownDirection) {
972
- const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
973
- if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
974
- }
975
- if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
976
- if (!screenSize) screenSize = await this.size();
977
- const pages = Math.max(1, Math.round(distance / screenSize.height));
978
- const key = 'up' === direction ? 'pageup' : 'pagedown';
979
- for(let i = 0; i < pages; i++){
980
- this.inputDriver.sendKeyViaAppleScript(key);
981
- await this.inputDriver.delay(SCROLL_STEP_DELAY);
982
- }
983
- await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
984
- return;
985
- }
986
- const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
987
- const directionUnit = {
988
- up: [
989
- 0,
990
- 1
991
- ],
992
- down: [
993
- 0,
994
- -1
995
- ],
996
- left: [
997
- -1,
998
- 0
999
- ],
1000
- right: [
1001
- 1,
1002
- 0
1003
- ]
1004
- };
1005
- const [ux, uy] = directionUnit[direction] || [
1006
- 0,
1007
- -1
1008
- ];
1009
- await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
1010
- await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
1011
- return;
1012
- }
1013
- throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
1014
- }
1015
- actionSpace() {
1016
- const defaultActions = [
1017
- ...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives)
1018
- ];
1019
- const platformActions = Object.values(createPlatformActions());
1020
- const customActions = this.options?.customActions || [];
1021
- return [
1022
- ...defaultActions,
1023
- ...platformActions,
1024
- ...customActions
1025
- ];
1026
- }
1027
- async destroy() {
1028
- if (this.destroyed) return;
1029
- this.destroyed = true;
1030
- this.inputDriver.destroy();
1031
- if (this.xvfbInstance) {
1032
- this.xvfbInstance.stop();
1033
- this.xvfbInstance = void 0;
1034
- }
1035
- if (this.xvfbCleanup) {
1036
- process.removeListener('exit', this.xvfbCleanup);
1037
- process.removeListener('SIGINT', this.xvfbCleanup);
1038
- process.removeListener('SIGTERM', this.xvfbCleanup);
1039
- this.xvfbCleanup = void 0;
1040
- }
1041
- debugDevice('Computer device destroyed');
1042
- }
1043
- async url() {
1044
- return '';
1045
- }
1046
- constructor(options){
1047
- device_define_property(this, "interfaceType", 'computer');
1048
- device_define_property(this, "options", void 0);
1049
- device_define_property(this, "displayId", void 0);
1050
- device_define_property(this, "displayGeometry", void 0);
1051
- device_define_property(this, "description", void 0);
1052
- device_define_property(this, "destroyed", false);
1053
- device_define_property(this, "xvfbInstance", void 0);
1054
- device_define_property(this, "xvfbCleanup", void 0);
1055
- device_define_property(this, "inputDriver", new ComputerInputDriver({
1056
- getLibnut: ()=>device_libnut,
1057
- useAppleScript: ()=>this.useAppleScript,
1058
- sendKeyViaAppleScript,
1059
- runPhasedScroll,
1060
- debug: (message)=>debugDevice(message)
1061
- }));
1062
- device_define_property(this, "useAppleScript", void 0);
1063
- device_define_property(this, "adminCheckCache", void 0);
1064
- device_define_property(this, "uri", void 0);
1065
- device_define_property(this, "inputPrimitives", {
1066
- pointer: {
1067
- tap: async ({ x, y }, opts)=>{
1068
- const target = this.toGlobalPoint({
1069
- x,
1070
- y
1071
- });
1072
- const targetX = Math.round(target.x);
1073
- const targetY = Math.round(target.y);
1074
- const holdDuration = Math.max(0, Math.round(opts?.duration ?? CLICK_HOLD_DURATION));
1075
- debugComputerInput('tap start %o', {
1076
- local: {
1077
- x,
1078
- y
1079
- },
1080
- global: {
1081
- x: targetX,
1082
- y: targetY
1083
- },
1084
- holdDuration,
1085
- displayId: this.displayId,
1086
- displayGeometry: this.displayGeometry ? {
1087
- screenIndex: this.displayGeometry.screenIndex,
1088
- cgDisplayId: this.displayGeometry.cgDisplayId,
1089
- bounds: this.displayGeometry.bounds
1090
- } : void 0
1091
- });
1092
- const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
1093
- await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
1094
- await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
1095
- if (frontmostBefore && 'darwin' === process.platform) {
1096
- await (0, utils_namespaceObject.sleep)(CLICK_FOCUS_SETTLE_DELAY);
1097
- const frontmostAfter = readDarwinFrontmostApplication();
1098
- const focusChanged = !!frontmostAfter && frontmostAfter.pid !== frontmostBefore.pid;
1099
- debugComputerInput('tap focus check %o', {
1100
- before: frontmostBefore,
1101
- after: frontmostAfter,
1102
- focusChanged
1103
- });
1104
- if (focusChanged) {
1105
- this.inputDriver.moveMouse(targetX, targetY);
1106
- await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
1107
- }
1108
- }
1109
- },
1110
- doubleClick: async ({ x, y })=>{
1111
- const target = this.toGlobalPoint({
1112
- x,
1113
- y
1114
- });
1115
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1116
- this.inputDriver.mouseClick('left', true);
1117
- },
1118
- rightClick: async ({ x, y })=>{
1119
- const target = this.toGlobalPoint({
1120
- x,
1121
- y
1122
- });
1123
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1124
- this.inputDriver.mouseClick('right');
1125
- },
1126
- hover: async ({ x, y })=>{
1127
- const target = this.toGlobalPoint({
1128
- x,
1129
- y
1130
- });
1131
- await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
1132
- await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
1133
- },
1134
- dragAndDrop: async (from, to)=>{
1135
- const globalFrom = this.toGlobalPoint(from);
1136
- const globalTo = this.toGlobalPoint(to);
1137
- this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
1138
- await this.inputDriver.withMouseButton('left', async ()=>{
1139
- await this.inputDriver.delay(100);
1140
- this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
1141
- await this.inputDriver.delay(100);
1142
- });
1143
- }
1144
- },
1145
- keyboard: {
1146
- typeText: async (value, opts)=>{
1147
- const element = opts?.target;
1148
- if (element) {
1149
- const [x, y] = element.center;
1150
- const target = this.toGlobalPoint({
1151
- x,
1152
- y
1153
- });
1154
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1155
- this.inputDriver.mouseClick('left');
1156
- await this.inputDriver.delay(INPUT_FOCUS_DELAY);
1157
- if (opts?.replace !== false) {
1158
- await this.selectAllAndDelete();
1159
- await this.inputDriver.delay(INPUT_CLEAR_DELAY);
1160
- }
1161
- }
1162
- await this.smartTypeString(value);
1163
- },
1164
- keyboardPress: async (keyName, opts)=>{
1165
- const target = opts?.target;
1166
- if (target) {
1167
- const [x, y] = target.center;
1168
- const point = this.toGlobalPoint({
1169
- x,
1170
- y
1171
- });
1172
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
1173
- this.inputDriver.mouseClick('left');
1174
- await this.inputDriver.delay(50);
1175
- }
1176
- await this.pressKeyboardShortcut(keyName);
1177
- },
1178
- clearInput: async (target)=>{
1179
- if (target) {
1180
- const element = target;
1181
- const [x, y] = element.center;
1182
- const point = this.toGlobalPoint({
1183
- x,
1184
- y
1185
- });
1186
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
1187
- this.inputDriver.mouseClick('left');
1188
- await this.inputDriver.delay(100);
1189
- }
1190
- await this.selectAllAndDelete();
1191
- await this.inputDriver.delay(50);
1192
- }
1193
- },
1194
- scroll: {
1195
- scroll: async (param)=>{
1196
- await this.performScroll(param);
1197
- }
1198
- }
1199
- });
1200
- this.options = options;
1201
- this.displayId = options?.displayId;
1202
- this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
1203
- }
1204
- }
1205
- function createPlatformActions() {
1206
- return {
1207
- ListDisplays: (0, device_namespaceObject.defineAction)({
1208
- name: 'ListDisplays',
1209
- description: 'List all available displays/monitors',
1210
- call: async ()=>await ComputerDevice.listDisplays()
1211
- })
1212
- };
1213
- }
1214
- function normalizeRdpHost(host) {
1215
- const trimmed = host.trim();
1216
- if (trimmed.length >= 2 && trimmed.startsWith('[') && trimmed.endsWith(']') && trimmed.includes(':')) return trimmed.slice(1, -1);
1217
- return trimmed;
1218
- }
1219
- function formatRdpHost(host) {
1220
- const normalizedHost = normalizeRdpHost(host);
1221
- return normalizedHost.includes(':') ? `[${normalizedHost}]` : normalizedHost;
1222
- }
1223
- function formatRdpServerAddress(host, port) {
1224
- return `${formatRdpHost(host)}:${port}`;
1225
- }
1226
- function normalizeRdpConnectionConfig(config) {
1227
- return {
1228
- ...config,
1229
- host: normalizeRdpHost(config.host),
1230
- ...config.localAddress ? {
1231
- localAddress: config.localAddress.trim()
1232
- } : {}
1233
- };
1234
- }
1235
- const external_node_events_namespaceObject = require("node:events");
1236
- const external_node_readline_namespaceObject = require("node:readline");
1237
- const platformBinaryMap = {
1238
- darwin: {
1239
- directory: 'darwin',
1240
- fileName: 'rdp-helper'
1241
- },
1242
- linux: {
1243
- directory: 'linux',
1244
- fileName: 'rdp-helper'
1245
- },
1246
- win32: {
1247
- directory: 'win32',
1248
- fileName: 'rdp-helper.exe'
1249
- }
1250
- };
1251
- function getPlatformBinary(platform) {
1252
- if (platform in platformBinaryMap) return platformBinaryMap[platform];
1253
- }
1254
- function currentDirname() {
1255
- return __dirname;
1256
- }
1257
- function getRdpHelperBinaryPath() {
1258
- const platformBinary = getPlatformBinary(process.platform);
1259
- if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
1260
- const hereDir = currentDirname();
1261
- const candidateRoots = [
1262
- (0, external_node_path_namespaceObject.resolve)(hereDir, '../..'),
1263
- (0, external_node_path_namespaceObject.resolve)(hereDir, '../../..')
1264
- ];
1265
- for (const root of candidateRoots){
1266
- const binaryPath = (0, external_node_path_namespaceObject.resolve)(root, 'bin', platformBinary.directory, platformBinary.fileName);
1267
- if ((0, external_node_fs_namespaceObject.existsSync)(binaryPath)) return binaryPath;
1268
- }
1269
- throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
1270
- }
1271
- function backend_client_define_property(obj, key, value) {
1272
- if (key in obj) Object.defineProperty(obj, key, {
1273
- value: value,
1274
- enumerable: true,
1275
- configurable: true,
1276
- writable: true
1277
- });
1278
- else obj[key] = value;
1279
- return obj;
1280
- }
1281
- const debug = (0, logger_namespaceObject.getDebug)('rdp:backend');
1282
- const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
1283
- const HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS = 50;
1284
- const MAX_STDERR_CHARS = 16384;
1285
- class HelperProcessRDPBackendClient {
1286
- async connect(config) {
1287
- this.fatalHelperError = void 0;
1288
- const response = await this.send({
1289
- type: 'connect',
1290
- config: normalizeRdpConnectionConfig(config)
1291
- });
1292
- if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
1293
- this.connected = true;
1294
- this.fatalHelperError = void 0;
1295
- return response.info;
1296
- }
1297
- async disconnect() {
1298
- const child = this.child;
1299
- if (!child) return;
1300
- let disconnectError;
1301
- if (this.connected && null === child.exitCode) try {
1302
- const response = await this.send({
1303
- type: 'disconnect'
1304
- });
1305
- this.expectOk(response, 'disconnect');
1306
- } catch (error) {
1307
- disconnectError = error instanceof Error ? error : new Error(String(error));
1308
- }
1309
- this.connected = false;
1310
- this.fatalHelperError = void 0;
1311
- await this.shutdownHelper();
1312
- if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
1313
- }
1314
- async screenshotBase64() {
1315
- const response = await this.send({
1316
- type: 'screenshot'
1317
- });
1318
- if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
1319
- return response.base64;
1320
- }
1321
- async size() {
1322
- const response = await this.send({
1323
- type: 'size'
1324
- });
1325
- if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
1326
- return response.size;
1327
- }
1328
- async mouseMove(x, y) {
1329
- const response = await this.send({
1330
- type: 'mouseMove',
1331
- x,
1332
- y
1333
- });
1334
- this.expectOk(response, 'mouseMove');
1335
- }
1336
- async mouseButton(button, action) {
1337
- const response = await this.send({
1338
- type: 'mouseButton',
1339
- button,
1340
- action
1341
- });
1342
- this.expectOk(response, 'mouseButton');
1343
- }
1344
- async wheel(direction, amount, x, y) {
1345
- const response = await this.send({
1346
- type: 'wheel',
1347
- direction,
1348
- amount,
1349
- x,
1350
- y
1351
- });
1352
- this.expectOk(response, 'wheel');
1353
- }
1354
- async keyPress(keyName) {
1355
- const response = await this.send({
1356
- type: 'keyPress',
1357
- keyName
1358
- });
1359
- this.expectOk(response, 'keyPress');
1360
- }
1361
- async typeText(text) {
1362
- const response = await this.send({
1363
- type: 'typeText',
1364
- text
1365
- });
1366
- this.expectOk(response, 'typeText');
1367
- }
1368
- async clearInput() {
1369
- const response = await this.send({
1370
- type: 'clearInput'
1371
- });
1372
- this.expectOk(response, 'clearInput');
1373
- }
1374
- ensureHelperStarted() {
1375
- if (this.child && null === this.child.exitCode) return this.child;
1376
- const helperPath = this.resolveHelperPath();
1377
- debug('starting rdp helper', {
1378
- helperPath
1379
- });
1380
- const child = this.spawnFn(helperPath, [], {
1381
- stdio: [
1382
- 'pipe',
1383
- 'pipe',
1384
- 'pipe'
1385
- ]
1386
- });
1387
- child.stdout.setEncoding('utf8');
1388
- child.stderr.setEncoding('utf8');
1389
- this.child = child;
1390
- const diagnostics = {
1391
- path: helperPath,
1392
- pid: child.pid,
1393
- stderrBuffer: ''
1394
- };
1395
- this.helperDiagnostics.set(child, diagnostics);
1396
- debug('started rdp helper', {
1397
- helperPath,
1398
- pid: child.pid
1399
- });
1400
- this.stdoutReader = (0, external_node_readline_namespaceObject.createInterface)({
1401
- input: child.stdout,
1402
- crlfDelay: 1 / 0
1403
- });
1404
- this.stdoutReader.on('line', (line)=>{
1405
- this.handleStdoutLine(child, line, diagnostics);
1406
- });
1407
- child.stderr.on('data', (chunk)=>{
1408
- this.captureStderrChunk(chunk, diagnostics);
1409
- });
1410
- child.stdin.on('error', (error)=>{
1411
- this.handleHelperStreamError(child, 'stdin', error, diagnostics);
1412
- });
1413
- child.stdout.on('error', (error)=>{
1414
- this.handleHelperStreamError(child, 'stdout', error, diagnostics);
1415
- });
1416
- child.stderr.on('error', (error)=>{
1417
- this.handleHelperStreamError(child, 'stderr', error, diagnostics);
1418
- });
1419
- child.on('exit', (code, signal)=>{
1420
- diagnostics.exit = {
1421
- code,
1422
- signal
1423
- };
1424
- debug('rdp helper exited', {
1425
- helperPath: diagnostics.path,
1426
- pid: diagnostics.pid,
1427
- code,
1428
- signal
1429
- });
1430
- const helperError = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`, void 0, diagnostics);
1431
- this.rejectPendingForChild(child, helperError);
1432
- if (this.child === child) {
1433
- this.connected = false;
1434
- this.fatalHelperError = helperError;
1435
- this.disposeReaders();
1436
- this.child = void 0;
1437
- }
1438
- });
1439
- child.on('error', (error)=>{
1440
- const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`, void 0, diagnostics);
1441
- this.rejectPendingForChild(child, helperError);
1442
- if (this.child === child) {
1443
- this.connected = false;
1444
- this.fatalHelperError = helperError;
1445
- this.disposeReaders();
1446
- this.child = void 0;
1447
- }
1448
- });
1449
- return child;
1450
- }
1451
- handleHelperStreamError(child, streamName, error, diagnostics) {
1452
- const nodeError = error;
1453
- const helperError = this.createHelperError(`RDP helper ${streamName} stream error: ${error.message}`, nodeError.code, diagnostics);
1454
- this.rejectPendingForChild(child, helperError);
1455
- if (this.child !== child) return;
1456
- this.connected = false;
1457
- this.fatalHelperError = helperError;
1458
- if (null === child.exitCode) child.kill('SIGTERM');
1459
- this.child = void 0;
1460
- }
1461
- handleStdoutLine(child, line, diagnostics) {
1462
- if (!line.trim()) return;
1463
- let parsed;
1464
- try {
1465
- parsed = JSON.parse(line);
1466
- } catch (error) {
1467
- const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`, void 0, diagnostics);
1468
- this.rejectPendingForChild(child, protocolError);
1469
- if (this.child === child) this.shutdownHelper(protocolError);
1470
- return;
1471
- }
1472
- const pending = this.pending.get(parsed.id);
1473
- if (!pending) return void debug('dropping response for unknown request id', parsed);
1474
- this.pending.delete(parsed.id);
1475
- if (parsed.ok) return void pending.resolve(parsed.payload);
1476
- pending.reject(this.createHelperError(parsed.error.message, parsed.error.code, diagnostics));
1477
- }
1478
- captureStderrChunk(chunk, diagnostics) {
1479
- const text = chunk.toString();
1480
- if (!text.trim()) return;
1481
- debug('rdp helper stderr', {
1482
- helperPath: diagnostics.path,
1483
- pid: diagnostics.pid,
1484
- stderr: text.trim()
1485
- });
1486
- diagnostics.stderrBuffer += text;
1487
- if (diagnostics.stderrBuffer.length > MAX_STDERR_CHARS) diagnostics.stderrBuffer = diagnostics.stderrBuffer.slice(-MAX_STDERR_CHARS);
1488
- }
1489
- async send(payload) {
1490
- if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
1491
- const child = this.ensureHelperStarted();
1492
- if (null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
1493
- const id = `req-${++this.nextRequestId}`;
1494
- const diagnostics = this.helperDiagnostics.get(child);
1495
- const request = {
1496
- id,
1497
- payload
1498
- };
1499
- return new Promise((resolve, reject)=>{
1500
- this.pending.set(id, {
1501
- resolve,
1502
- reject,
1503
- child
1504
- });
1505
- child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
1506
- if (!error) return;
1507
- this.pending.delete(id);
1508
- const nodeError = error;
1509
- const timer = setTimeout(()=>{
1510
- reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`, nodeError.code, diagnostics));
1511
- }, HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS);
1512
- timer.unref?.();
1513
- });
1514
- });
1515
- }
1516
- expectOk(response, actionName) {
1517
- if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
1518
- }
1519
- rejectPending(error) {
1520
- for (const { reject } of this.pending.values())reject(error);
1521
- this.pending.clear();
1522
- }
1523
- rejectPendingForChild(child, error) {
1524
- for (const [id, pending] of this.pending)if (pending.child === child) {
1525
- pending.reject(error);
1526
- this.pending.delete(id);
1527
- }
1528
- }
1529
- createHelperError(message, code, diagnostics) {
1530
- const diagnosticParts = [
1531
- diagnostics?.path ? `path=${diagnostics.path}` : void 0,
1532
- 'number' == typeof diagnostics?.pid ? `pid=${diagnostics.pid}` : void 0,
1533
- diagnostics?.exit ? `exitCode=${diagnostics.exit.code}, signal=${diagnostics.exit.signal}` : void 0
1534
- ].filter(Boolean);
1535
- const diagnosticsSuffix = diagnosticParts.length > 0 ? `\nHelper diagnostics: ${diagnosticParts.join(', ')}` : '';
1536
- const stderrSummary = diagnostics?.stderrBuffer.trim();
1537
- const stderrSuffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
1538
- const error = new Error(`${message}${diagnosticsSuffix}${stderrSuffix}`);
1539
- if (code) error.name = code;
1540
- return error;
1541
- }
1542
- disposeReaders() {
1543
- this.stdoutReader?.close();
1544
- this.stdoutReader = void 0;
1545
- }
1546
- async shutdownHelper(rootError) {
1547
- const child = this.child;
1548
- this.child = void 0;
1549
- this.disposeReaders();
1550
- if (!child) return;
1551
- this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
1552
- if (null !== child.exitCode) return;
1553
- child.stdin.end();
1554
- const exited = Promise.race([
1555
- (0, external_node_events_namespaceObject.once)(child, 'exit'),
1556
- new Promise((resolve)=>{
1557
- setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1558
- })
1559
- ]);
1560
- const result = await exited;
1561
- if ('timeout' !== result) return;
1562
- child.kill('SIGTERM');
1563
- const terminated = Promise.race([
1564
- (0, external_node_events_namespaceObject.once)(child, 'exit'),
1565
- new Promise((resolve)=>{
1566
- setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1567
- })
1568
- ]);
1569
- const terminateResult = await terminated;
1570
- if ('timeout' !== terminateResult) return;
1571
- child.kill('SIGKILL');
1572
- await (0, external_node_events_namespaceObject.once)(child, 'exit');
1573
- }
1574
- constructor(options){
1575
- backend_client_define_property(this, "spawnFn", void 0);
1576
- backend_client_define_property(this, "resolveHelperPath", void 0);
1577
- backend_client_define_property(this, "child", void 0);
1578
- backend_client_define_property(this, "stdoutReader", void 0);
1579
- backend_client_define_property(this, "pending", new Map());
1580
- backend_client_define_property(this, "helperDiagnostics", new WeakMap());
1581
- backend_client_define_property(this, "nextRequestId", 0);
1582
- backend_client_define_property(this, "connected", false);
1583
- backend_client_define_property(this, "fatalHelperError", void 0);
1584
- this.spawnFn = options?.spawnFn || external_node_child_process_namespaceObject.spawn;
1585
- const overridePath = options?.helperPath;
1586
- this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
1587
- }
1588
- }
1589
- function createDefaultRDPBackendClient() {
1590
- return new HelperProcessRDPBackendClient();
1591
- }
1592
- function rdp_device_define_property(obj, key, value) {
1593
- if (key in obj) Object.defineProperty(obj, key, {
1594
- value: value,
1595
- enumerable: true,
1596
- configurable: true,
1597
- writable: true
1598
- });
1599
- else obj[key] = value;
1600
- return obj;
1601
- }
1602
- const device_debug = (0, logger_namespaceObject.getDebug)('rdp:device');
1603
- const device_SMOOTH_MOVE_STEPS_TAP = 8;
1604
- const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
1605
- const SMOOTH_MOVE_STEPS_DRAG = 12;
1606
- const device_SMOOTH_MOVE_DELAY_TAP = 8;
1607
- const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
1608
- const SMOOTH_MOVE_DELAY_DRAG = 10;
1609
- const device_MOUSE_MOVE_EFFECT_WAIT = 300;
1610
- const device_CLICK_HOLD_DURATION = 50;
1611
- const DRAG_HOLD_DURATION = 100;
1612
- const device_INPUT_FOCUS_DELAY = 300;
1613
- const device_INPUT_CLEAR_DELAY = 150;
1614
- const device_SCROLL_STEP_DELAY = 100;
1615
- const device_SCROLL_COMPLETE_DELAY = 500;
1616
- const DEFAULT_SCROLL_DISTANCE = 480;
1617
- const device_DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
1618
- const device_EDGE_SCROLL_STEPS = 10;
1619
- const DEFAULT_SCROLL_STEP_AMOUNT = 120;
1620
- class RDPDevice {
1621
- describe() {
1622
- const port = this.options.port || 3389;
1623
- const server = formatRdpServerAddress(this.options.host, port);
1624
- const username = this.options.username ? ` as ${this.options.username}` : '';
1625
- const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
1626
- return `RDP Device ${server}${username}${session}`;
1627
- }
1628
- async connect() {
1629
- this.throwIfDestroyed();
1630
- device_debug('connecting to rdp backend', {
1631
- host: this.options.host,
1632
- port: this.options.port,
1633
- username: this.options.username
1634
- });
1635
- const { backend: _backend, customActions: _customActions, ...config } = this.options;
1636
- this.connectionInfo = await this.backend.connect(config);
1637
- this.cursorPosition = [
1638
- Math.round(this.connectionInfo.size.width / 2),
1639
- Math.round(this.connectionInfo.size.height / 2)
1640
- ];
1641
- }
1642
- async screenshotBase64() {
1643
- this.assertConnected();
1644
- return this.backend.screenshotBase64();
1645
- }
1646
- async size() {
1647
- this.assertConnected();
1648
- return this.backend.size();
1649
- }
1650
- async destroy() {
1651
- if (this.destroyed) return;
1652
- this.destroyed = true;
1653
- this.connectionInfo = void 0;
1654
- this.cursorPosition = void 0;
1655
- await this.backend.disconnect();
1656
- }
1657
- actionSpace() {
1658
- const defaultActions = [
1659
- ...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives),
1660
- (0, device_namespaceObject.defineAction)({
1661
- name: 'ListDisplays',
1662
- description: 'List all available displays/monitors',
1663
- call: async ()=>{
1664
- this.assertConnected();
1665
- const size = await this.size();
1666
- const server = this.connectionInfo?.server || formatRdpServerAddress(this.options.host, this.options.port || 3389);
1667
- return [
1668
- {
1669
- id: this.connectionInfo?.sessionId || this.options.host,
1670
- name: `RDP ${server} (${size.width}x${size.height})`,
1671
- primary: true
1672
- }
1673
- ];
1674
- }
1675
- })
1676
- ];
1677
- return [
1678
- ...defaultActions,
1679
- ...this.options.customActions || []
1680
- ];
1681
- }
1682
- assertConnected() {
1683
- this.throwIfDestroyed();
1684
- if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
1685
- }
1686
- throwIfDestroyed() {
1687
- if (this.destroyed) throw new Error('RDPDevice has been destroyed');
1688
- }
1689
- async moveToElement(element, options) {
1690
- this.assertConnected();
1691
- const targetX = Math.round(element.center[0]);
1692
- const targetY = Math.round(element.center[1]);
1693
- await this.movePointer(targetX, targetY, options);
1694
- }
1695
- async clearInput() {
1696
- if (this.backend.clearInput) return void await this.backend.clearInput();
1697
- await this.backend.keyPress('Control+A');
1698
- await this.backend.keyPress('Backspace');
1699
- }
1700
- edgeScrollDirection(scrollType) {
1701
- switch(scrollType){
1702
- case 'scrollToTop':
1703
- return 'up';
1704
- case 'scrollToBottom':
1705
- return 'down';
1706
- case 'scrollToLeft':
1707
- return 'left';
1708
- case 'scrollToRight':
1709
- return 'right';
1710
- case 'singleAction':
1711
- return 'down';
1712
- default:
1713
- throw new Error(`Unsupported scroll type: ${scrollType}`);
1714
- }
1715
- }
1716
- defaultScrollDistance(direction) {
1717
- const size = this.connectionInfo?.size;
1718
- if (!size) return DEFAULT_SCROLL_DISTANCE;
1719
- const isHorizontal = 'left' === direction || 'right' === direction;
1720
- const base = isHorizontal ? size.width : size.height;
1721
- return Math.max(1, Math.round(base * device_DEFAULT_SCROLL_VIEWPORT_RATIO));
1722
- }
1723
- async movePointer(targetX, targetY, options) {
1724
- this.assertConnected();
1725
- const start = this.cursorPosition || [
1726
- targetX,
1727
- targetY
1728
- ];
1729
- const steps = Math.max(1, options?.steps || 1);
1730
- const stepDelayMs = options?.stepDelayMs || 0;
1731
- for(let step = 1; step <= steps; step++){
1732
- const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
1733
- const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
1734
- await this.backend.mouseMove(x, y);
1735
- this.cursorPosition = [
1736
- x,
1737
- y
1738
- ];
1739
- if (stepDelayMs > 0 && step < steps) await (0, utils_namespaceObject.sleep)(stepDelayMs);
1740
- }
1741
- if (options?.settleDelayMs) await (0, utils_namespaceObject.sleep)(options.settleDelayMs);
1742
- }
1743
- async performWheel(direction, amount, x, y) {
1744
- let remaining = Math.abs(amount);
1745
- if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
1746
- while(remaining > 0){
1747
- const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
1748
- await this.backend.wheel(direction, chunk, x, y);
1749
- remaining -= chunk;
1750
- if (remaining > 0) await (0, utils_namespaceObject.sleep)(device_SCROLL_STEP_DELAY);
1751
- }
1752
- }
1753
- constructor(options){
1754
- rdp_device_define_property(this, "interfaceType", 'rdp');
1755
- rdp_device_define_property(this, "options", void 0);
1756
- rdp_device_define_property(this, "backend", void 0);
1757
- rdp_device_define_property(this, "connectionInfo", void 0);
1758
- rdp_device_define_property(this, "destroyed", false);
1759
- rdp_device_define_property(this, "cursorPosition", void 0);
1760
- rdp_device_define_property(this, "uri", void 0);
1761
- rdp_device_define_property(this, "inputPrimitives", {
1762
- pointer: {
1763
- tap: async ({ x, y })=>{
1764
- await this.movePointer(Math.round(x), Math.round(y), {
1765
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1766
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1767
- });
1768
- await this.backend.mouseButton('left', 'down');
1769
- await (0, utils_namespaceObject.sleep)(device_CLICK_HOLD_DURATION);
1770
- await this.backend.mouseButton('left', 'up');
1771
- },
1772
- doubleClick: async ({ x, y })=>{
1773
- await this.movePointer(Math.round(x), Math.round(y), {
1774
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1775
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1776
- });
1777
- await this.backend.mouseButton('left', 'doubleClick');
1778
- },
1779
- rightClick: async ({ x, y })=>{
1780
- await this.movePointer(Math.round(x), Math.round(y), {
1781
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1782
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1783
- });
1784
- await this.backend.mouseButton('right', 'click');
1785
- },
1786
- hover: async ({ x, y })=>{
1787
- await this.movePointer(Math.round(x), Math.round(y), {
1788
- steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1789
- stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
1790
- settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
1791
- });
1792
- },
1793
- dragAndDrop: async (from, to)=>{
1794
- await this.movePointer(Math.round(from.x), Math.round(from.y), {
1795
- steps: device_SMOOTH_MOVE_STEPS_TAP,
1796
- stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1797
- });
1798
- await this.backend.mouseButton('left', 'down');
1799
- await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
1800
- await this.movePointer(Math.round(to.x), Math.round(to.y), {
1801
- steps: SMOOTH_MOVE_STEPS_DRAG,
1802
- stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
1803
- });
1804
- await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
1805
- await this.backend.mouseButton('left', 'up');
1806
- }
1807
- },
1808
- keyboard: {
1809
- typeText: async (value, opts)=>{
1810
- this.assertConnected();
1811
- const target = opts?.target;
1812
- if (target) {
1813
- await this.inputPrimitives.pointer.tap({
1814
- x: target.center[0],
1815
- y: target.center[1]
1816
- });
1817
- await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
1818
- }
1819
- if (opts?.replace !== false) {
1820
- await this.clearInput();
1821
- await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
1822
- }
1823
- if (opts?.focusOnly || !value) return;
1824
- await this.backend.typeText(value);
1825
- },
1826
- clearInput: async (target)=>{
1827
- this.assertConnected();
1828
- const element = target;
1829
- if (element) {
1830
- await this.inputPrimitives.pointer.tap({
1831
- x: element.center[0],
1832
- y: element.center[1]
1833
- });
1834
- await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
1835
- }
1836
- await this.clearInput();
1837
- await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
1838
- },
1839
- keyboardPress: async (keyName, opts)=>{
1840
- this.assertConnected();
1841
- const target = opts?.target;
1842
- if (target) await this.inputPrimitives.pointer.tap({
1843
- x: target.center[0],
1844
- y: target.center[1]
1845
- });
1846
- await this.backend.keyPress(keyName);
1847
- }
1848
- },
1849
- scroll: {
1850
- scroll: async (param)=>{
1851
- this.assertConnected();
1852
- const target = param.locate;
1853
- if (target) await this.moveToElement(target, {
1854
- steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1855
- stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
1856
- });
1857
- if (param.scrollType && 'singleAction' !== param.scrollType) {
1858
- const direction = this.edgeScrollDirection(param.scrollType);
1859
- for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
1860
- await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
1861
- return;
1862
- }
1863
- await this.performWheel(param.direction || 'down', param.distance || this.defaultScrollDistance(param.direction || 'down'), target?.center[0], target?.center[1]);
1864
- await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
1865
- }
1866
- }
1867
- });
1868
- const normalizedOptions = normalizeRdpConnectionConfig(options);
1869
- this.options = {
1870
- port: 3389,
1871
- securityProtocol: 'auto',
1872
- ignoreCertificate: false,
1873
- ...normalizedOptions
1874
- };
1875
- this.backend = options.backend || createDefaultRDPBackendClient();
1876
- }
1877
- }
1878
- class ComputerAgent extends agent_namespaceObject.Agent {
1879
- }
1880
- function createLocalComputerDevice(opts) {
1881
- return new ComputerDevice({
1882
- displayId: opts?.displayId,
1883
- customActions: opts?.customActions,
1884
- keyboardDriver: opts?.keyboardDriver,
1885
- headless: opts?.headless,
1886
- xvfbResolution: opts?.xvfbResolution
1887
- });
1888
- }
1889
- function createRDPComputerDevice(opts) {
1890
- return new RDPDevice({
1891
- host: opts.host,
1892
- port: opts.port,
1893
- username: opts.username,
1894
- password: opts.password,
1895
- domain: opts.domain,
1896
- localAddress: opts.localAddress,
1897
- adminSession: opts.adminSession,
1898
- ignoreCertificate: opts.ignoreCertificate,
1899
- securityProtocol: opts.securityProtocol,
1900
- desktopWidth: opts.desktopWidth,
1901
- desktopHeight: opts.desktopHeight,
1902
- backend: opts.backend,
1903
- customActions: opts.customActions
1904
- });
1905
- }
1906
- async function agentForComputer(opts) {
1907
- const device = createLocalComputerDevice(opts);
1908
- await device.connect();
1909
- return new ComputerAgent(device, opts);
1910
- }
1911
- const agentFromComputer = agentForComputer;
1912
- async function agentForRDPComputer(opts) {
1913
- const device = createRDPComputerDevice(opts);
1914
- await device.connect();
1915
- return new ComputerAgent(device, opts);
1916
- }
1917
- const core_namespaceObject = require("@midscene/core");
1918
- const agent_behavior_init_args_namespaceObject = require("@midscene/shared/mcp/agent-behavior-init-args");
1919
- const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
1920
- function mcp_tools_define_property(obj, key, value) {
1921
- if (key in obj) Object.defineProperty(obj, key, {
1922
- value: value,
1923
- enumerable: true,
1924
- configurable: true,
1925
- writable: true
1926
- });
1927
- else obj[key] = value;
1928
- return obj;
1929
- }
1930
- const mcp_tools_debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
1931
- const RDP_SECURITY_PROTOCOLS = [
1932
- 'auto',
1933
- 'tls',
1934
- 'nla',
1935
- 'rdp'
1936
- ];
1937
- const computerInitArgShape = {
1938
- displayId: core_namespaceObject.z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
1939
- headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
1940
- host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
1941
- port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
1942
- username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
1943
- password: core_namespaceObject.z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
1944
- domain: core_namespaceObject.z.string().optional().describe('RDP domain. Requires host.'),
1945
- localAddress: core_namespaceObject.z.string().optional().describe('Local source IP address for the RDP TCP connection. Requires host.'),
1946
- adminSession: core_namespaceObject.z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
1947
- ignoreCertificate: core_namespaceObject.z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
1948
- securityProtocol: core_namespaceObject.z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
1949
- desktopWidth: core_namespaceObject.z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
1950
- desktopHeight: core_namespaceObject.z.number().optional().describe('Remote desktop height in pixels. Requires host.'),
1951
- ...agent_behavior_init_args_namespaceObject.agentBehaviorInitArgShape
1952
- };
1953
- function adaptComputerInitArgs(extracted) {
1954
- if (!extracted || 0 === Object.keys(extracted).length) return;
1955
- if (extracted.host) {
1956
- const { displayId: _d, headless: _h, ...rdpFields } = extracted;
1957
- const host = normalizeRdpHost(extracted.host);
1958
- return {
1959
- mode: 'rdp',
1960
- ...rdpFields,
1961
- host
1962
- };
1963
- }
1964
- return {
1965
- mode: 'local',
1966
- displayId: extracted.displayId,
1967
- headless: extracted.headless,
1968
- ...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(extracted) ?? {}
1969
- };
1970
- }
1971
- function describeConnectTarget(opts) {
1972
- if (opts?.mode === 'rdp') {
1973
- const target = opts.port ? formatRdpServerAddress(opts.host, opts.port) : formatRdpHost(opts.host);
1974
- const userSuffix = opts.username ? ` as ${opts.username}` : '';
1975
- return ` via RDP (${target}${userSuffix})`;
1976
- }
1977
- if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
1978
- return ' (Primary display)';
1979
- }
1980
- function getCliReportSessionTarget(opts) {
1981
- if (opts?.mode === 'rdp') return `rdp:${formatRdpHost(opts.host)}`;
1982
- if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
1983
- return 'primary';
1984
- }
1985
- class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
1986
- getCliReportSessionName() {
1987
- return 'midscene-computer';
1988
- }
1989
- createTemporaryDevice() {
1990
- return new ComputerDevice({});
1991
- }
1992
- async ensureAgent(opts) {
1993
- const nextSignature = (0, agent_behavior_init_args_namespaceObject.getAgentInitArgsSignature)(opts);
1994
- if (this.agent && (0, agent_behavior_init_args_namespaceObject.shouldRebuildAgentForInitArgs)(this.lastInitArgsSignature, nextSignature)) {
1995
- try {
1996
- await this.agent.destroy?.();
1997
- } catch (error) {
1998
- mcp_tools_debug('Failed to destroy agent during cleanup:', error);
1999
- }
2000
- this.agent = void 0;
2001
- }
2002
- if (this.agent) return this.agent;
2003
- const reportOptions = this.readCliReportAgentOptions();
2004
- if (opts?.mode === 'rdp') {
2005
- mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
2006
- const { mode: _mode, ...rdpFields } = opts;
2007
- const agent = await agentForRDPComputer({
2008
- ...rdpFields,
2009
- ...reportOptions ?? {}
2010
- });
2011
- this.agent = agent;
2012
- this.lastInitArgsSignature = nextSignature;
2013
- return agent;
2014
- }
2015
- const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
2016
- const headless = opts?.mode === 'local' ? opts.headless : void 0;
2017
- mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
2018
- const agentOpts = {
2019
- ...displayId ? {
2020
- displayId
2021
- } : {},
2022
- ...void 0 !== headless ? {
2023
- headless
2024
- } : {},
2025
- ...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(opts) ?? {},
2026
- ...reportOptions ?? {}
2027
- };
2028
- const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
2029
- this.agent = agent;
2030
- this.lastInitArgsSignature = nextSignature;
2031
- return agent;
2032
- }
2033
- preparePlatformTools() {
2034
- return [
2035
- {
2036
- name: 'computer_connect',
2037
- description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/localAddress/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
2038
- schema: this.getAgentInitArgSchema(),
2039
- cli: this.getAgentInitArgCliMetadata(),
2040
- handler: async (args)=>{
2041
- const initArgs = this.extractAgentInitParam(args);
2042
- const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
2043
- this.commitCliReportSession(reportSession);
2044
- if (this.agent) {
2045
- try {
2046
- await this.agent.destroy?.();
2047
- } catch (error) {
2048
- mcp_tools_debug('Failed to destroy agent during connect:', error);
2049
- }
2050
- this.agent = void 0;
2051
- this.lastInitArgsSignature = void 0;
2052
- }
2053
- const agent = await this.ensureAgent(initArgs);
2054
- const screenshot = await agent.interface.screenshotBase64();
2055
- return {
2056
- content: [
2057
- {
2058
- type: 'text',
2059
- text: `Connected to computer${describeConnectTarget(initArgs)}`
2060
- },
2061
- ...this.buildScreenshotContent(screenshot)
2062
- ]
2063
- };
2064
- }
2065
- },
2066
- {
2067
- name: 'computer_disconnect',
2068
- description: 'Disconnect from computer and release resources',
2069
- schema: {},
2070
- handler: this.createDisconnectHandler('computer')
2071
- },
2072
- {
2073
- name: 'computer_list_displays',
2074
- description: 'List all available displays/monitors',
2075
- schema: {},
2076
- handler: async ()=>{
2077
- const displays = await ComputerDevice.listDisplays();
2078
- return {
2079
- content: [
2080
- {
2081
- type: 'text',
2082
- text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
2083
- }
2084
- ]
2085
- };
2086
- }
2087
- }
2088
- ];
2089
- }
2090
- constructor(...args){
2091
- super(...args), mcp_tools_define_property(this, "lastInitArgsSignature", void 0), mcp_tools_define_property(this, "initArgSpec", {
2092
- namespace: 'computer',
2093
- shape: computerInitArgShape,
2094
- cli: {
2095
- preferBareKeys: true
2096
- },
2097
- adapt: (extracted)=>adaptComputerInitArgs(extracted)
2098
- });
2099
- }
2100
- }
2101
- class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
2102
- createToolsManager() {
2103
- return new ComputerMidsceneTools();
2104
- }
2105
- constructor(toolsManager){
2106
- super({
2107
- name: '@midscene/computer-mcp',
2108
- version: "1.9.8",
2109
- description: 'Control the computer desktop using natural language commands'
2110
- }, toolsManager);
2111
- }
2112
- }
2113
- function mcpServerForAgent(agent) {
2114
- return (0, mcp_namespaceObject.createMCPServerLauncher)({
2115
- agent,
2116
- platformName: 'Computer',
2117
- ToolsManagerClass: ComputerMidsceneTools,
2118
- MCPServerClass: ComputerMCPServer
2119
- });
2120
- }
2121
- async function mcpKitForAgent(agent) {
2122
- const toolsManager = new ComputerMidsceneTools();
2123
- const computerAgent = agent instanceof ComputerAgent ? agent : agent;
2124
- toolsManager.setAgent(computerAgent);
2125
- await toolsManager.initTools();
2126
- return {
2127
- description: 'Midscene MCP Kit for computer desktop automation',
2128
- tools: toolsManager.getToolDefinitions()
2129
- };
2130
- }
2131
- exports.ComputerMCPServer = __webpack_exports__.ComputerMCPServer;
2132
- exports.mcpKitForAgent = __webpack_exports__.mcpKitForAgent;
2133
- exports.mcpServerForAgent = __webpack_exports__.mcpServerForAgent;
2134
- for(var __rspack_i in __webpack_exports__)if (-1 === [
2135
- "ComputerMCPServer",
2136
- "mcpKitForAgent",
2137
- "mcpServerForAgent"
2138
- ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
2139
- Object.defineProperty(exports, '__esModule', {
2140
- value: true
2141
- });