@midscene/computer 1.8.6 → 1.8.7-beta-20260527113633.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +221 -112
- package/dist/es/index.mjs +215 -106
- package/dist/es/mcp-server.mjs +221 -112
- package/dist/lib/cli.js +222 -113
- package/dist/lib/index.js +216 -107
- package/dist/lib/mcp-server.js +222 -113
- package/dist/types/index.d.ts +1 -0
- package/dist/types/mcp-server.d.ts +1 -0
- package/package.json +3 -3
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
|
|
2
2
|
import { Agent } from "@midscene/core/agent";
|
|
3
|
-
import node_assert from "node:assert";
|
|
4
3
|
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
5
4
|
import { existsSync } from "node:fs";
|
|
6
5
|
import { createRequire } from "node:module";
|
|
@@ -11,10 +10,139 @@ import { sleep } from "@midscene/core/utils";
|
|
|
11
10
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
12
11
|
import { getDebug } from "@midscene/shared/logger";
|
|
13
12
|
import screenshot_desktop from "screenshot-desktop";
|
|
13
|
+
import node_assert from "node:assert";
|
|
14
14
|
import { once } from "node:events";
|
|
15
15
|
import { createInterface } from "node:readline";
|
|
16
16
|
import { z } from "@midscene/core";
|
|
17
17
|
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
18
|
+
function _define_property(obj, key, value) {
|
|
19
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
20
|
+
value: value,
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true
|
|
24
|
+
});
|
|
25
|
+
else obj[key] = value;
|
|
26
|
+
return obj;
|
|
27
|
+
}
|
|
28
|
+
class ComputerInputDriver {
|
|
29
|
+
destroy() {
|
|
30
|
+
if (this.destroyed) return;
|
|
31
|
+
this.destroyed = true;
|
|
32
|
+
this.rejectPendingInputDelays();
|
|
33
|
+
}
|
|
34
|
+
getScreenSize() {
|
|
35
|
+
return this.getLibnutOrThrow('getScreenSize').getScreenSize();
|
|
36
|
+
}
|
|
37
|
+
getMousePos() {
|
|
38
|
+
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
39
|
+
}
|
|
40
|
+
moveMouse(x, y) {
|
|
41
|
+
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
42
|
+
}
|
|
43
|
+
mouseClick(button, double) {
|
|
44
|
+
const lib = this.getLibnutOrThrow('mouseClick');
|
|
45
|
+
if (void 0 !== double) lib.mouseClick(button, double);
|
|
46
|
+
else if (void 0 !== button) lib.mouseClick(button);
|
|
47
|
+
else lib.mouseClick();
|
|
48
|
+
}
|
|
49
|
+
mouseToggle(state, button = 'left') {
|
|
50
|
+
this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
|
|
51
|
+
}
|
|
52
|
+
scrollMouse(x, y) {
|
|
53
|
+
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
|
+
}
|
|
55
|
+
keyTap(key, modifiers) {
|
|
56
|
+
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
|
+
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
58
|
+
else lib.keyTap(key);
|
|
59
|
+
}
|
|
60
|
+
sendKeyViaAppleScript(key, modifiers = []) {
|
|
61
|
+
this.assertActive('sendKeyViaAppleScript');
|
|
62
|
+
this.options.sendKeyViaAppleScript(key, modifiers);
|
|
63
|
+
}
|
|
64
|
+
sendKey(key, modifiers = []) {
|
|
65
|
+
if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
|
|
66
|
+
if (modifiers.length > 0) this.keyTap(key, modifiers);
|
|
67
|
+
else this.keyTap(key);
|
|
68
|
+
}
|
|
69
|
+
runPhasedScroll(direction, pixels, steps) {
|
|
70
|
+
this.assertActive('runPhasedScroll');
|
|
71
|
+
return this.options.runPhasedScroll(direction, pixels, steps);
|
|
72
|
+
}
|
|
73
|
+
async delay(ms) {
|
|
74
|
+
this.assertActive('delay');
|
|
75
|
+
return new Promise((resolve, reject)=>{
|
|
76
|
+
const waitRef = {
|
|
77
|
+
timeoutId: setTimeout(()=>{
|
|
78
|
+
this.pendingInputDelayWaits.delete(waitRef);
|
|
79
|
+
try {
|
|
80
|
+
this.assertActive('delay');
|
|
81
|
+
resolve();
|
|
82
|
+
} catch (error) {
|
|
83
|
+
reject(error);
|
|
84
|
+
}
|
|
85
|
+
}, ms),
|
|
86
|
+
reject
|
|
87
|
+
};
|
|
88
|
+
this.pendingInputDelayWaits.add(waitRef);
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
92
|
+
const currentPos = this.getMousePos();
|
|
93
|
+
for(let i = 1; i <= steps; i++){
|
|
94
|
+
const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
|
|
95
|
+
const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
|
|
96
|
+
this.moveMouse(stepX, stepY);
|
|
97
|
+
await this.delay(stepDelay);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async withMouseButton(button, run) {
|
|
101
|
+
this.mouseToggle('down', button);
|
|
102
|
+
try {
|
|
103
|
+
return await run();
|
|
104
|
+
} finally{
|
|
105
|
+
this.releaseMouseButton(button);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
getLibnutOrThrow(methodName) {
|
|
109
|
+
this.assertActive(methodName);
|
|
110
|
+
const libnut = this.options.getLibnut();
|
|
111
|
+
node_assert(libnut, 'libnut not initialized');
|
|
112
|
+
return libnut;
|
|
113
|
+
}
|
|
114
|
+
assertActive(methodName) {
|
|
115
|
+
if (this.destroyed) throw this.createDestroyedError(methodName);
|
|
116
|
+
}
|
|
117
|
+
createDestroyedError(methodName) {
|
|
118
|
+
return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
|
|
119
|
+
}
|
|
120
|
+
releaseMouseButton(button) {
|
|
121
|
+
try {
|
|
122
|
+
const libnut = this.options.getLibnut();
|
|
123
|
+
node_assert(libnut, 'libnut not initialized');
|
|
124
|
+
libnut.mouseToggle('up', button);
|
|
125
|
+
} catch (error) {
|
|
126
|
+
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
rejectPendingInputDelays() {
|
|
130
|
+
const error = this.createDestroyedError('in-flight input');
|
|
131
|
+
for (const waitRef of this.pendingInputDelayWaits){
|
|
132
|
+
clearTimeout(waitRef.timeoutId);
|
|
133
|
+
waitRef.reject(error);
|
|
134
|
+
}
|
|
135
|
+
this.pendingInputDelayWaits.clear();
|
|
136
|
+
}
|
|
137
|
+
constructor(options){
|
|
138
|
+
_define_property(this, "options", void 0);
|
|
139
|
+
_define_property(this, "destroyed", void 0);
|
|
140
|
+
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
141
|
+
this.options = options;
|
|
142
|
+
this.destroyed = false;
|
|
143
|
+
this.pendingInputDelayWaits = new Set();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
18
146
|
const debugXvfb = getDebug('computer:xvfb');
|
|
19
147
|
function checkXvfbInstalled() {
|
|
20
148
|
try {
|
|
@@ -82,7 +210,7 @@ function needsXvfb(explicitOpt) {
|
|
|
82
210
|
if ('linux' !== process.platform) return false;
|
|
83
211
|
return true === explicitOpt;
|
|
84
212
|
}
|
|
85
|
-
function
|
|
213
|
+
function device_define_property(obj, key, value) {
|
|
86
214
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
87
215
|
value: value,
|
|
88
216
|
enumerable: true,
|
|
@@ -203,17 +331,17 @@ function sendKeyViaAppleScript(key, modifiers = []) {
|
|
|
203
331
|
script
|
|
204
332
|
]);
|
|
205
333
|
}
|
|
206
|
-
let
|
|
334
|
+
let device_libnut = null;
|
|
207
335
|
let libnutLoadError = null;
|
|
208
336
|
async function getLibnut() {
|
|
209
|
-
if (
|
|
337
|
+
if (device_libnut) return device_libnut;
|
|
210
338
|
if (libnutLoadError) throw libnutLoadError;
|
|
211
339
|
try {
|
|
212
340
|
const require = createRequire(import.meta.url);
|
|
213
341
|
const libnutModule = require('@computer-use/libnut/dist/import_libnut');
|
|
214
|
-
|
|
215
|
-
if (!
|
|
216
|
-
return
|
|
342
|
+
device_libnut = libnutModule.libnut;
|
|
343
|
+
if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
|
|
344
|
+
return device_libnut;
|
|
217
345
|
} catch (error) {
|
|
218
346
|
libnutLoadError = error;
|
|
219
347
|
throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
|
|
@@ -283,16 +411,6 @@ function runPhasedScroll(direction, pixels, steps) {
|
|
|
283
411
|
return false;
|
|
284
412
|
}
|
|
285
413
|
}
|
|
286
|
-
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
287
|
-
node_assert(libnut, 'libnut not initialized');
|
|
288
|
-
const currentPos = libnut.getMousePos();
|
|
289
|
-
for(let i = 1; i <= steps; i++){
|
|
290
|
-
const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
|
|
291
|
-
const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
|
|
292
|
-
libnut.moveMouse(stepX, stepY);
|
|
293
|
-
await sleep(stepDelay);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
414
|
const KEY_NAME_MAP = {
|
|
297
415
|
windows: 'win',
|
|
298
416
|
win: 'win',
|
|
@@ -378,7 +496,7 @@ class ComputerDevice {
|
|
|
378
496
|
process.on('SIGINT', this.xvfbCleanup);
|
|
379
497
|
process.on('SIGTERM', this.xvfbCleanup);
|
|
380
498
|
}
|
|
381
|
-
|
|
499
|
+
device_libnut = await getLibnut();
|
|
382
500
|
const size = await this.size();
|
|
383
501
|
const displays = await ComputerDevice.listDisplays();
|
|
384
502
|
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
@@ -402,7 +520,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
402
520
|
}
|
|
403
521
|
async healthCheck() {
|
|
404
522
|
console.log('[HealthCheck] Starting health check...');
|
|
405
|
-
console.log("[HealthCheck] @midscene/computer v1.8.
|
|
523
|
+
console.log("[HealthCheck] @midscene/computer v1.8.7-beta-20260527113633.0");
|
|
406
524
|
console.log('[HealthCheck] Taking screenshot...');
|
|
407
525
|
const screenshotTimeout = 15000;
|
|
408
526
|
let timeoutId;
|
|
@@ -415,17 +533,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
415
533
|
]);
|
|
416
534
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
417
535
|
console.log('[HealthCheck] Moving mouse...');
|
|
418
|
-
|
|
419
|
-
const startPos = libnut.getMousePos();
|
|
536
|
+
const startPos = this.inputDriver.getMousePos();
|
|
420
537
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
421
538
|
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
422
539
|
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
423
540
|
const targetX = startPos.x + offsetX;
|
|
424
541
|
const targetY = startPos.y + offsetY;
|
|
425
542
|
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
426
|
-
|
|
543
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
427
544
|
await sleep(50);
|
|
428
|
-
const movedPos =
|
|
545
|
+
const movedPos = this.inputDriver.getMousePos();
|
|
429
546
|
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
430
547
|
const deltaX = Math.abs(movedPos.x - targetX);
|
|
431
548
|
const deltaY = Math.abs(movedPos.y - targetY);
|
|
@@ -439,7 +556,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
439
556
|
debugDevice(hint);
|
|
440
557
|
}
|
|
441
558
|
}
|
|
442
|
-
|
|
559
|
+
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
443
560
|
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
444
561
|
console.log('[HealthCheck] Listing monitors...');
|
|
445
562
|
const displays = await ComputerDevice.listDisplays();
|
|
@@ -502,9 +619,8 @@ Original error: ${lastRawMessage}`);
|
|
|
502
619
|
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
503
620
|
}
|
|
504
621
|
async size() {
|
|
505
|
-
node_assert(libnut, 'libnut not initialized');
|
|
506
622
|
try {
|
|
507
|
-
const screenSize =
|
|
623
|
+
const screenSize = this.inputDriver.getScreenSize();
|
|
508
624
|
return {
|
|
509
625
|
width: screenSize.width,
|
|
510
626
|
height: screenSize.height
|
|
@@ -515,7 +631,6 @@ Original error: ${lastRawMessage}`);
|
|
|
515
631
|
}
|
|
516
632
|
}
|
|
517
633
|
async typeViaClipboard(text) {
|
|
518
|
-
node_assert(libnut, 'libnut not initialized');
|
|
519
634
|
debugDevice('Using clipboard to input text', {
|
|
520
635
|
textLength: text.length,
|
|
521
636
|
preview: text.substring(0, 20)
|
|
@@ -524,17 +639,17 @@ Original error: ${lastRawMessage}`);
|
|
|
524
639
|
const oldClipboard = await clipboardy.default.read().catch(()=>'');
|
|
525
640
|
try {
|
|
526
641
|
await clipboardy.default.write(text);
|
|
527
|
-
await
|
|
528
|
-
if (this.useAppleScript) sendKeyViaAppleScript('v', [
|
|
642
|
+
await this.inputDriver.delay(50);
|
|
643
|
+
if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
|
|
529
644
|
'command'
|
|
530
645
|
]);
|
|
531
646
|
else {
|
|
532
647
|
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
533
|
-
|
|
648
|
+
this.inputDriver.keyTap('v', [
|
|
534
649
|
modifier
|
|
535
650
|
]);
|
|
536
651
|
}
|
|
537
|
-
await
|
|
652
|
+
await this.inputDriver.delay(100);
|
|
538
653
|
} finally{
|
|
539
654
|
if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
|
|
540
655
|
debugDevice('Failed to restore clipboard content');
|
|
@@ -542,28 +657,25 @@ Original error: ${lastRawMessage}`);
|
|
|
542
657
|
}
|
|
543
658
|
}
|
|
544
659
|
async smartTypeString(text) {
|
|
545
|
-
node_assert(libnut, 'libnut not initialized');
|
|
546
660
|
await this.typeViaClipboard(text);
|
|
547
661
|
}
|
|
548
662
|
async selectAllAndDelete() {
|
|
549
|
-
node_assert(libnut, 'libnut not initialized');
|
|
550
663
|
if (this.useAppleScript) {
|
|
551
|
-
sendKeyViaAppleScript('a', [
|
|
664
|
+
this.inputDriver.sendKeyViaAppleScript('a', [
|
|
552
665
|
'command'
|
|
553
666
|
]);
|
|
554
|
-
await
|
|
555
|
-
sendKeyViaAppleScript('backspace', []);
|
|
667
|
+
await this.inputDriver.delay(50);
|
|
668
|
+
this.inputDriver.sendKeyViaAppleScript('backspace', []);
|
|
556
669
|
return;
|
|
557
670
|
}
|
|
558
671
|
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
559
|
-
|
|
672
|
+
this.inputDriver.keyTap('a', [
|
|
560
673
|
modifier
|
|
561
674
|
]);
|
|
562
|
-
await
|
|
563
|
-
|
|
675
|
+
await this.inputDriver.delay(50);
|
|
676
|
+
this.inputDriver.keyTap('backspace');
|
|
564
677
|
}
|
|
565
678
|
async pressKeyboardShortcut(keyName) {
|
|
566
|
-
node_assert(libnut, 'libnut not initialized');
|
|
567
679
|
const keys = keyName.split('+');
|
|
568
680
|
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
569
681
|
const key = normalizePrimaryKey(keys[keys.length - 1]);
|
|
@@ -573,30 +685,27 @@ Original error: ${lastRawMessage}`);
|
|
|
573
685
|
modifiers,
|
|
574
686
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
575
687
|
});
|
|
576
|
-
|
|
577
|
-
else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
|
|
578
|
-
else libnut.keyTap(key);
|
|
688
|
+
this.inputDriver.sendKey(key, modifiers);
|
|
579
689
|
}
|
|
580
690
|
async performScroll(param) {
|
|
581
|
-
node_assert(libnut, 'libnut not initialized');
|
|
582
691
|
if (param.locate) {
|
|
583
692
|
const element = param.locate;
|
|
584
693
|
const [x, y] = element.center;
|
|
585
|
-
|
|
694
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
586
695
|
}
|
|
587
696
|
const scrollType = param?.scrollType;
|
|
588
697
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
589
698
|
if (edgeSpec) {
|
|
590
|
-
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await
|
|
699
|
+
if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
591
700
|
if (this.useAppleScript) {
|
|
592
|
-
sendKeyViaAppleScript(edgeSpec.key);
|
|
593
|
-
await
|
|
701
|
+
this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
|
|
702
|
+
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
594
703
|
return;
|
|
595
704
|
}
|
|
596
705
|
const [dx, dy] = edgeSpec.libnut;
|
|
597
706
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
598
|
-
|
|
599
|
-
await
|
|
707
|
+
this.inputDriver.scrollMouse(dx, dy);
|
|
708
|
+
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
600
709
|
}
|
|
601
710
|
return;
|
|
602
711
|
}
|
|
@@ -613,17 +722,17 @@ Original error: ${lastRawMessage}`);
|
|
|
613
722
|
}
|
|
614
723
|
if (isKnownDirection) {
|
|
615
724
|
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
616
|
-
if (runPhasedScroll(direction, distance, steps)) return void await
|
|
725
|
+
if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
617
726
|
}
|
|
618
727
|
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
619
728
|
if (!screenSize) screenSize = await this.size();
|
|
620
729
|
const pages = Math.max(1, Math.round(distance / screenSize.height));
|
|
621
730
|
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
622
731
|
for(let i = 0; i < pages; i++){
|
|
623
|
-
sendKeyViaAppleScript(key);
|
|
624
|
-
await
|
|
732
|
+
this.inputDriver.sendKeyViaAppleScript(key);
|
|
733
|
+
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
625
734
|
}
|
|
626
|
-
await
|
|
735
|
+
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
627
736
|
return;
|
|
628
737
|
}
|
|
629
738
|
const ticks = Math.ceil(distance / 100);
|
|
@@ -649,8 +758,8 @@ Original error: ${lastRawMessage}`);
|
|
|
649
758
|
0,
|
|
650
759
|
-ticks
|
|
651
760
|
];
|
|
652
|
-
|
|
653
|
-
await
|
|
761
|
+
this.inputDriver.scrollMouse(dx, dy);
|
|
762
|
+
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
654
763
|
return;
|
|
655
764
|
}
|
|
656
765
|
throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
|
|
@@ -669,6 +778,8 @@ Original error: ${lastRawMessage}`);
|
|
|
669
778
|
}
|
|
670
779
|
async destroy() {
|
|
671
780
|
if (this.destroyed) return;
|
|
781
|
+
this.destroyed = true;
|
|
782
|
+
this.inputDriver.destroy();
|
|
672
783
|
if (this.xvfbInstance) {
|
|
673
784
|
this.xvfbInstance.stop();
|
|
674
785
|
this.xvfbInstance = void 0;
|
|
@@ -679,96 +790,94 @@ Original error: ${lastRawMessage}`);
|
|
|
679
790
|
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
680
791
|
this.xvfbCleanup = void 0;
|
|
681
792
|
}
|
|
682
|
-
this.destroyed = true;
|
|
683
793
|
debugDevice('Computer device destroyed');
|
|
684
794
|
}
|
|
685
795
|
async url() {
|
|
686
796
|
return '';
|
|
687
797
|
}
|
|
688
798
|
constructor(options){
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
799
|
+
device_define_property(this, "interfaceType", 'computer');
|
|
800
|
+
device_define_property(this, "options", void 0);
|
|
801
|
+
device_define_property(this, "displayId", void 0);
|
|
802
|
+
device_define_property(this, "description", void 0);
|
|
803
|
+
device_define_property(this, "destroyed", false);
|
|
804
|
+
device_define_property(this, "xvfbInstance", void 0);
|
|
805
|
+
device_define_property(this, "xvfbCleanup", void 0);
|
|
806
|
+
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
807
|
+
getLibnut: ()=>device_libnut,
|
|
808
|
+
useAppleScript: ()=>this.useAppleScript,
|
|
809
|
+
sendKeyViaAppleScript,
|
|
810
|
+
runPhasedScroll,
|
|
811
|
+
debug: (message)=>debugDevice(message)
|
|
812
|
+
}));
|
|
813
|
+
device_define_property(this, "useAppleScript", void 0);
|
|
814
|
+
device_define_property(this, "uri", void 0);
|
|
815
|
+
device_define_property(this, "inputPrimitives", {
|
|
699
816
|
pointer: {
|
|
700
817
|
tap: async ({ x, y })=>{
|
|
701
|
-
node_assert(libnut, 'libnut not initialized');
|
|
702
818
|
const targetX = Math.round(x);
|
|
703
819
|
const targetY = Math.round(y);
|
|
704
|
-
await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
820
|
+
await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
821
|
+
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
822
|
+
await this.inputDriver.delay(CLICK_HOLD_DURATION);
|
|
823
|
+
});
|
|
708
824
|
},
|
|
709
825
|
doubleClick: async ({ x, y })=>{
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
libnut.mouseClick('left', true);
|
|
826
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
827
|
+
this.inputDriver.mouseClick('left', true);
|
|
713
828
|
},
|
|
714
829
|
rightClick: async ({ x, y })=>{
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
libnut.mouseClick('right');
|
|
830
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
831
|
+
this.inputDriver.mouseClick('right');
|
|
718
832
|
},
|
|
719
833
|
hover: async ({ x, y })=>{
|
|
720
|
-
|
|
721
|
-
await
|
|
722
|
-
await sleep(MOUSE_MOVE_EFFECT_WAIT);
|
|
834
|
+
await this.inputDriver.smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
835
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
723
836
|
},
|
|
724
837
|
dragAndDrop: async (from, to)=>{
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
libnut.mouseToggle('up', 'left');
|
|
838
|
+
this.inputDriver.moveMouse(Math.round(from.x), Math.round(from.y));
|
|
839
|
+
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
840
|
+
await this.inputDriver.delay(100);
|
|
841
|
+
this.inputDriver.moveMouse(Math.round(to.x), Math.round(to.y));
|
|
842
|
+
await this.inputDriver.delay(100);
|
|
843
|
+
});
|
|
732
844
|
}
|
|
733
845
|
},
|
|
734
846
|
keyboard: {
|
|
735
847
|
typeText: async (value, opts)=>{
|
|
736
|
-
node_assert(libnut, 'libnut not initialized');
|
|
737
848
|
const element = opts?.target;
|
|
738
849
|
if (element) {
|
|
739
850
|
const [x, y] = element.center;
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
await
|
|
851
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
852
|
+
this.inputDriver.mouseClick('left');
|
|
853
|
+
await this.inputDriver.delay(INPUT_FOCUS_DELAY);
|
|
743
854
|
if (opts?.replace !== false) {
|
|
744
855
|
await this.selectAllAndDelete();
|
|
745
|
-
await
|
|
856
|
+
await this.inputDriver.delay(INPUT_CLEAR_DELAY);
|
|
746
857
|
}
|
|
747
858
|
}
|
|
748
859
|
await this.smartTypeString(value);
|
|
749
860
|
},
|
|
750
861
|
keyboardPress: async (keyName, opts)=>{
|
|
751
|
-
node_assert(libnut, 'libnut not initialized');
|
|
752
862
|
const target = opts?.target;
|
|
753
863
|
if (target) {
|
|
754
864
|
const [x, y] = target.center;
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
await
|
|
865
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
866
|
+
this.inputDriver.mouseClick('left');
|
|
867
|
+
await this.inputDriver.delay(50);
|
|
758
868
|
}
|
|
759
869
|
await this.pressKeyboardShortcut(keyName);
|
|
760
870
|
},
|
|
761
871
|
clearInput: async (target)=>{
|
|
762
|
-
node_assert(libnut, 'libnut not initialized');
|
|
763
872
|
if (target) {
|
|
764
873
|
const element = target;
|
|
765
874
|
const [x, y] = element.center;
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
await
|
|
875
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
876
|
+
this.inputDriver.mouseClick('left');
|
|
877
|
+
await this.inputDriver.delay(100);
|
|
769
878
|
}
|
|
770
879
|
await this.selectAllAndDelete();
|
|
771
|
-
await
|
|
880
|
+
await this.inputDriver.delay(50);
|
|
772
881
|
}
|
|
773
882
|
},
|
|
774
883
|
scroll: {
|
|
@@ -1088,7 +1197,7 @@ class HelperProcessRDPBackendClient {
|
|
|
1088
1197
|
function createDefaultRDPBackendClient() {
|
|
1089
1198
|
return new HelperProcessRDPBackendClient();
|
|
1090
1199
|
}
|
|
1091
|
-
function
|
|
1200
|
+
function rdp_device_define_property(obj, key, value) {
|
|
1092
1201
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
1093
1202
|
value: value,
|
|
1094
1203
|
enumerable: true,
|
|
@@ -1247,14 +1356,14 @@ class RDPDevice {
|
|
|
1247
1356
|
}
|
|
1248
1357
|
}
|
|
1249
1358
|
constructor(options){
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1359
|
+
rdp_device_define_property(this, "interfaceType", 'rdp');
|
|
1360
|
+
rdp_device_define_property(this, "options", void 0);
|
|
1361
|
+
rdp_device_define_property(this, "backend", void 0);
|
|
1362
|
+
rdp_device_define_property(this, "connectionInfo", void 0);
|
|
1363
|
+
rdp_device_define_property(this, "destroyed", false);
|
|
1364
|
+
rdp_device_define_property(this, "cursorPosition", void 0);
|
|
1365
|
+
rdp_device_define_property(this, "uri", void 0);
|
|
1366
|
+
rdp_device_define_property(this, "inputPrimitives", {
|
|
1258
1367
|
pointer: {
|
|
1259
1368
|
tap: async ({ x, y })=>{
|
|
1260
1369
|
await this.movePointer(Math.round(x), Math.round(y), {
|
|
@@ -1592,7 +1701,7 @@ class ComputerMCPServer extends BaseMCPServer {
|
|
|
1592
1701
|
constructor(toolsManager){
|
|
1593
1702
|
super({
|
|
1594
1703
|
name: '@midscene/computer-mcp',
|
|
1595
|
-
version: "1.8.
|
|
1704
|
+
version: "1.8.7-beta-20260527113633.0",
|
|
1596
1705
|
description: 'Control the computer desktop using natural language commands'
|
|
1597
1706
|
}, toolsManager);
|
|
1598
1707
|
}
|