@midscene/computer 1.8.6 → 1.8.7-beta-20260527113633.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +221 -112
- package/dist/es/index.mjs +215 -106
- package/dist/es/mcp-server.mjs +221 -112
- package/dist/lib/cli.js +222 -113
- package/dist/lib/index.js +216 -107
- package/dist/lib/mcp-server.js +222 -113
- package/dist/types/index.d.ts +1 -0
- package/dist/types/mcp-server.d.ts +1 -0
- package/package.json +3 -3
package/dist/lib/mcp-server.js
CHANGED
|
@@ -42,8 +42,6 @@ __webpack_require__.d(__webpack_exports__, {
|
|
|
42
42
|
});
|
|
43
43
|
const mcp_namespaceObject = require("@midscene/shared/mcp");
|
|
44
44
|
const agent_namespaceObject = require("@midscene/core/agent");
|
|
45
|
-
const external_node_assert_namespaceObject = require("node:assert");
|
|
46
|
-
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
47
45
|
const external_node_child_process_namespaceObject = require("node:child_process");
|
|
48
46
|
const external_node_fs_namespaceObject = require("node:fs");
|
|
49
47
|
const external_node_module_namespaceObject = require("node:module");
|
|
@@ -55,6 +53,136 @@ const img_namespaceObject = require("@midscene/shared/img");
|
|
|
55
53
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
56
54
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
57
55
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
56
|
+
const external_node_assert_namespaceObject = require("node:assert");
|
|
57
|
+
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
58
|
+
function _define_property(obj, key, value) {
|
|
59
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
60
|
+
value: value,
|
|
61
|
+
enumerable: true,
|
|
62
|
+
configurable: true,
|
|
63
|
+
writable: true
|
|
64
|
+
});
|
|
65
|
+
else obj[key] = value;
|
|
66
|
+
return obj;
|
|
67
|
+
}
|
|
68
|
+
class ComputerInputDriver {
|
|
69
|
+
destroy() {
|
|
70
|
+
if (this.destroyed) return;
|
|
71
|
+
this.destroyed = true;
|
|
72
|
+
this.rejectPendingInputDelays();
|
|
73
|
+
}
|
|
74
|
+
getScreenSize() {
|
|
75
|
+
return this.getLibnutOrThrow('getScreenSize').getScreenSize();
|
|
76
|
+
}
|
|
77
|
+
getMousePos() {
|
|
78
|
+
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
79
|
+
}
|
|
80
|
+
moveMouse(x, y) {
|
|
81
|
+
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
82
|
+
}
|
|
83
|
+
mouseClick(button, double) {
|
|
84
|
+
const lib = this.getLibnutOrThrow('mouseClick');
|
|
85
|
+
if (void 0 !== double) lib.mouseClick(button, double);
|
|
86
|
+
else if (void 0 !== button) lib.mouseClick(button);
|
|
87
|
+
else lib.mouseClick();
|
|
88
|
+
}
|
|
89
|
+
mouseToggle(state, button = 'left') {
|
|
90
|
+
this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
|
|
91
|
+
}
|
|
92
|
+
scrollMouse(x, y) {
|
|
93
|
+
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
94
|
+
}
|
|
95
|
+
keyTap(key, modifiers) {
|
|
96
|
+
const lib = this.getLibnutOrThrow('keyTap');
|
|
97
|
+
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
98
|
+
else lib.keyTap(key);
|
|
99
|
+
}
|
|
100
|
+
sendKeyViaAppleScript(key, modifiers = []) {
|
|
101
|
+
this.assertActive('sendKeyViaAppleScript');
|
|
102
|
+
this.options.sendKeyViaAppleScript(key, modifiers);
|
|
103
|
+
}
|
|
104
|
+
sendKey(key, modifiers = []) {
|
|
105
|
+
if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
|
|
106
|
+
if (modifiers.length > 0) this.keyTap(key, modifiers);
|
|
107
|
+
else this.keyTap(key);
|
|
108
|
+
}
|
|
109
|
+
runPhasedScroll(direction, pixels, steps) {
|
|
110
|
+
this.assertActive('runPhasedScroll');
|
|
111
|
+
return this.options.runPhasedScroll(direction, pixels, steps);
|
|
112
|
+
}
|
|
113
|
+
async delay(ms) {
|
|
114
|
+
this.assertActive('delay');
|
|
115
|
+
return new Promise((resolve, reject)=>{
|
|
116
|
+
const waitRef = {
|
|
117
|
+
timeoutId: setTimeout(()=>{
|
|
118
|
+
this.pendingInputDelayWaits.delete(waitRef);
|
|
119
|
+
try {
|
|
120
|
+
this.assertActive('delay');
|
|
121
|
+
resolve();
|
|
122
|
+
} catch (error) {
|
|
123
|
+
reject(error);
|
|
124
|
+
}
|
|
125
|
+
}, ms),
|
|
126
|
+
reject
|
|
127
|
+
};
|
|
128
|
+
this.pendingInputDelayWaits.add(waitRef);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
132
|
+
const currentPos = this.getMousePos();
|
|
133
|
+
for(let i = 1; i <= steps; i++){
|
|
134
|
+
const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
|
|
135
|
+
const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
|
|
136
|
+
this.moveMouse(stepX, stepY);
|
|
137
|
+
await this.delay(stepDelay);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async withMouseButton(button, run) {
|
|
141
|
+
this.mouseToggle('down', button);
|
|
142
|
+
try {
|
|
143
|
+
return await run();
|
|
144
|
+
} finally{
|
|
145
|
+
this.releaseMouseButton(button);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
getLibnutOrThrow(methodName) {
|
|
149
|
+
this.assertActive(methodName);
|
|
150
|
+
const libnut = this.options.getLibnut();
|
|
151
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
152
|
+
return libnut;
|
|
153
|
+
}
|
|
154
|
+
assertActive(methodName) {
|
|
155
|
+
if (this.destroyed) throw this.createDestroyedError(methodName);
|
|
156
|
+
}
|
|
157
|
+
createDestroyedError(methodName) {
|
|
158
|
+
return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
|
|
159
|
+
}
|
|
160
|
+
releaseMouseButton(button) {
|
|
161
|
+
try {
|
|
162
|
+
const libnut = this.options.getLibnut();
|
|
163
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
164
|
+
libnut.mouseToggle('up', button);
|
|
165
|
+
} catch (error) {
|
|
166
|
+
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
rejectPendingInputDelays() {
|
|
170
|
+
const error = this.createDestroyedError('in-flight input');
|
|
171
|
+
for (const waitRef of this.pendingInputDelayWaits){
|
|
172
|
+
clearTimeout(waitRef.timeoutId);
|
|
173
|
+
waitRef.reject(error);
|
|
174
|
+
}
|
|
175
|
+
this.pendingInputDelayWaits.clear();
|
|
176
|
+
}
|
|
177
|
+
constructor(options){
|
|
178
|
+
_define_property(this, "options", void 0);
|
|
179
|
+
_define_property(this, "destroyed", void 0);
|
|
180
|
+
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
181
|
+
this.options = options;
|
|
182
|
+
this.destroyed = false;
|
|
183
|
+
this.pendingInputDelayWaits = new Set();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
58
186
|
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
59
187
|
function checkXvfbInstalled() {
|
|
60
188
|
try {
|
|
@@ -122,7 +250,7 @@ function needsXvfb(explicitOpt) {
|
|
|
122
250
|
if ('linux' !== process.platform) return false;
|
|
123
251
|
return true === explicitOpt;
|
|
124
252
|
}
|
|
125
|
-
function
|
|
253
|
+
function device_define_property(obj, key, value) {
|
|
126
254
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
127
255
|
value: value,
|
|
128
256
|
enumerable: true,
|
|
@@ -243,17 +371,17 @@ function sendKeyViaAppleScript(key, modifiers = []) {
|
|
|
243
371
|
script
|
|
244
372
|
]);
|
|
245
373
|
}
|
|
246
|
-
let
|
|
374
|
+
let device_libnut = null;
|
|
247
375
|
let libnutLoadError = null;
|
|
248
376
|
async function getLibnut() {
|
|
249
|
-
if (
|
|
377
|
+
if (device_libnut) return device_libnut;
|
|
250
378
|
if (libnutLoadError) throw libnutLoadError;
|
|
251
379
|
try {
|
|
252
380
|
const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
|
|
253
381
|
const libnutModule = require1('@computer-use/libnut/dist/import_libnut');
|
|
254
|
-
|
|
255
|
-
if (!
|
|
256
|
-
return
|
|
382
|
+
device_libnut = libnutModule.libnut;
|
|
383
|
+
if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
|
|
384
|
+
return device_libnut;
|
|
257
385
|
} catch (error) {
|
|
258
386
|
libnutLoadError = error;
|
|
259
387
|
throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
|
|
@@ -323,16 +451,6 @@ function runPhasedScroll(direction, pixels, steps) {
|
|
|
323
451
|
return false;
|
|
324
452
|
}
|
|
325
453
|
}
|
|
326
|
-
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
327
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
328
|
-
const currentPos = libnut.getMousePos();
|
|
329
|
-
for(let i = 1; i <= steps; i++){
|
|
330
|
-
const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
|
|
331
|
-
const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
|
|
332
|
-
libnut.moveMouse(stepX, stepY);
|
|
333
|
-
await (0, utils_namespaceObject.sleep)(stepDelay);
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
454
|
const KEY_NAME_MAP = {
|
|
337
455
|
windows: 'win',
|
|
338
456
|
win: 'win',
|
|
@@ -418,7 +536,7 @@ class ComputerDevice {
|
|
|
418
536
|
process.on('SIGINT', this.xvfbCleanup);
|
|
419
537
|
process.on('SIGTERM', this.xvfbCleanup);
|
|
420
538
|
}
|
|
421
|
-
|
|
539
|
+
device_libnut = await getLibnut();
|
|
422
540
|
const size = await this.size();
|
|
423
541
|
const displays = await ComputerDevice.listDisplays();
|
|
424
542
|
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
@@ -442,7 +560,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
442
560
|
}
|
|
443
561
|
async healthCheck() {
|
|
444
562
|
console.log('[HealthCheck] Starting health check...');
|
|
445
|
-
console.log("[HealthCheck] @midscene/computer v1.8.
|
|
563
|
+
console.log("[HealthCheck] @midscene/computer v1.8.7-beta-20260527113633.0");
|
|
446
564
|
console.log('[HealthCheck] Taking screenshot...');
|
|
447
565
|
const screenshotTimeout = 15000;
|
|
448
566
|
let timeoutId;
|
|
@@ -455,17 +573,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
455
573
|
]);
|
|
456
574
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
457
575
|
console.log('[HealthCheck] Moving mouse...');
|
|
458
|
-
|
|
459
|
-
const startPos = libnut.getMousePos();
|
|
576
|
+
const startPos = this.inputDriver.getMousePos();
|
|
460
577
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
461
578
|
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
462
579
|
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
463
580
|
const targetX = startPos.x + offsetX;
|
|
464
581
|
const targetY = startPos.y + offsetY;
|
|
465
582
|
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
466
|
-
|
|
583
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
467
584
|
await (0, utils_namespaceObject.sleep)(50);
|
|
468
|
-
const movedPos =
|
|
585
|
+
const movedPos = this.inputDriver.getMousePos();
|
|
469
586
|
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
470
587
|
const deltaX = Math.abs(movedPos.x - targetX);
|
|
471
588
|
const deltaY = Math.abs(movedPos.y - targetY);
|
|
@@ -479,7 +596,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
479
596
|
debugDevice(hint);
|
|
480
597
|
}
|
|
481
598
|
}
|
|
482
|
-
|
|
599
|
+
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
483
600
|
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
484
601
|
console.log('[HealthCheck] Listing monitors...');
|
|
485
602
|
const displays = await ComputerDevice.listDisplays();
|
|
@@ -542,9 +659,8 @@ Original error: ${lastRawMessage}`);
|
|
|
542
659
|
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
543
660
|
}
|
|
544
661
|
async size() {
|
|
545
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
546
662
|
try {
|
|
547
|
-
const screenSize =
|
|
663
|
+
const screenSize = this.inputDriver.getScreenSize();
|
|
548
664
|
return {
|
|
549
665
|
width: screenSize.width,
|
|
550
666
|
height: screenSize.height
|
|
@@ -555,7 +671,6 @@ Original error: ${lastRawMessage}`);
|
|
|
555
671
|
}
|
|
556
672
|
}
|
|
557
673
|
async typeViaClipboard(text) {
|
|
558
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
559
674
|
debugDevice('Using clipboard to input text', {
|
|
560
675
|
textLength: text.length,
|
|
561
676
|
preview: text.substring(0, 20)
|
|
@@ -564,17 +679,17 @@ Original error: ${lastRawMessage}`);
|
|
|
564
679
|
const oldClipboard = await clipboardy.default.read().catch(()=>'');
|
|
565
680
|
try {
|
|
566
681
|
await clipboardy.default.write(text);
|
|
567
|
-
await
|
|
568
|
-
if (this.useAppleScript) sendKeyViaAppleScript('v', [
|
|
682
|
+
await this.inputDriver.delay(50);
|
|
683
|
+
if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
|
|
569
684
|
'command'
|
|
570
685
|
]);
|
|
571
686
|
else {
|
|
572
687
|
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
573
|
-
|
|
688
|
+
this.inputDriver.keyTap('v', [
|
|
574
689
|
modifier
|
|
575
690
|
]);
|
|
576
691
|
}
|
|
577
|
-
await
|
|
692
|
+
await this.inputDriver.delay(100);
|
|
578
693
|
} finally{
|
|
579
694
|
if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
|
|
580
695
|
debugDevice('Failed to restore clipboard content');
|
|
@@ -582,28 +697,25 @@ Original error: ${lastRawMessage}`);
|
|
|
582
697
|
}
|
|
583
698
|
}
|
|
584
699
|
async smartTypeString(text) {
|
|
585
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
586
700
|
await this.typeViaClipboard(text);
|
|
587
701
|
}
|
|
588
702
|
async selectAllAndDelete() {
|
|
589
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
590
703
|
if (this.useAppleScript) {
|
|
591
|
-
sendKeyViaAppleScript('a', [
|
|
704
|
+
this.inputDriver.sendKeyViaAppleScript('a', [
|
|
592
705
|
'command'
|
|
593
706
|
]);
|
|
594
|
-
await
|
|
595
|
-
sendKeyViaAppleScript('backspace', []);
|
|
707
|
+
await this.inputDriver.delay(50);
|
|
708
|
+
this.inputDriver.sendKeyViaAppleScript('backspace', []);
|
|
596
709
|
return;
|
|
597
710
|
}
|
|
598
711
|
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
599
|
-
|
|
712
|
+
this.inputDriver.keyTap('a', [
|
|
600
713
|
modifier
|
|
601
714
|
]);
|
|
602
|
-
await
|
|
603
|
-
|
|
715
|
+
await this.inputDriver.delay(50);
|
|
716
|
+
this.inputDriver.keyTap('backspace');
|
|
604
717
|
}
|
|
605
718
|
async pressKeyboardShortcut(keyName) {
|
|
606
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
607
719
|
const keys = keyName.split('+');
|
|
608
720
|
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
609
721
|
const key = normalizePrimaryKey(keys[keys.length - 1]);
|
|
@@ -613,30 +725,27 @@ Original error: ${lastRawMessage}`);
|
|
|
613
725
|
modifiers,
|
|
614
726
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
615
727
|
});
|
|
616
|
-
|
|
617
|
-
else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
|
|
618
|
-
else libnut.keyTap(key);
|
|
728
|
+
this.inputDriver.sendKey(key, modifiers);
|
|
619
729
|
}
|
|
620
730
|
async performScroll(param) {
|
|
621
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
622
731
|
if (param.locate) {
|
|
623
732
|
const element = param.locate;
|
|
624
733
|
const [x, y] = element.center;
|
|
625
|
-
|
|
734
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
626
735
|
}
|
|
627
736
|
const scrollType = param?.scrollType;
|
|
628
737
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
629
738
|
if (edgeSpec) {
|
|
630
|
-
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await
|
|
739
|
+
if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
631
740
|
if (this.useAppleScript) {
|
|
632
|
-
sendKeyViaAppleScript(edgeSpec.key);
|
|
633
|
-
await
|
|
741
|
+
this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
|
|
742
|
+
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
634
743
|
return;
|
|
635
744
|
}
|
|
636
745
|
const [dx, dy] = edgeSpec.libnut;
|
|
637
746
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
638
|
-
|
|
639
|
-
await
|
|
747
|
+
this.inputDriver.scrollMouse(dx, dy);
|
|
748
|
+
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
640
749
|
}
|
|
641
750
|
return;
|
|
642
751
|
}
|
|
@@ -653,17 +762,17 @@ Original error: ${lastRawMessage}`);
|
|
|
653
762
|
}
|
|
654
763
|
if (isKnownDirection) {
|
|
655
764
|
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
656
|
-
if (runPhasedScroll(direction, distance, steps)) return void await
|
|
765
|
+
if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
657
766
|
}
|
|
658
767
|
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
659
768
|
if (!screenSize) screenSize = await this.size();
|
|
660
769
|
const pages = Math.max(1, Math.round(distance / screenSize.height));
|
|
661
770
|
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
662
771
|
for(let i = 0; i < pages; i++){
|
|
663
|
-
sendKeyViaAppleScript(key);
|
|
664
|
-
await
|
|
772
|
+
this.inputDriver.sendKeyViaAppleScript(key);
|
|
773
|
+
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
665
774
|
}
|
|
666
|
-
await
|
|
775
|
+
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
667
776
|
return;
|
|
668
777
|
}
|
|
669
778
|
const ticks = Math.ceil(distance / 100);
|
|
@@ -689,8 +798,8 @@ Original error: ${lastRawMessage}`);
|
|
|
689
798
|
0,
|
|
690
799
|
-ticks
|
|
691
800
|
];
|
|
692
|
-
|
|
693
|
-
await
|
|
801
|
+
this.inputDriver.scrollMouse(dx, dy);
|
|
802
|
+
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
694
803
|
return;
|
|
695
804
|
}
|
|
696
805
|
throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
|
|
@@ -709,6 +818,8 @@ Original error: ${lastRawMessage}`);
|
|
|
709
818
|
}
|
|
710
819
|
async destroy() {
|
|
711
820
|
if (this.destroyed) return;
|
|
821
|
+
this.destroyed = true;
|
|
822
|
+
this.inputDriver.destroy();
|
|
712
823
|
if (this.xvfbInstance) {
|
|
713
824
|
this.xvfbInstance.stop();
|
|
714
825
|
this.xvfbInstance = void 0;
|
|
@@ -719,96 +830,94 @@ Original error: ${lastRawMessage}`);
|
|
|
719
830
|
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
720
831
|
this.xvfbCleanup = void 0;
|
|
721
832
|
}
|
|
722
|
-
this.destroyed = true;
|
|
723
833
|
debugDevice('Computer device destroyed');
|
|
724
834
|
}
|
|
725
835
|
async url() {
|
|
726
836
|
return '';
|
|
727
837
|
}
|
|
728
838
|
constructor(options){
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
839
|
+
device_define_property(this, "interfaceType", 'computer');
|
|
840
|
+
device_define_property(this, "options", void 0);
|
|
841
|
+
device_define_property(this, "displayId", void 0);
|
|
842
|
+
device_define_property(this, "description", void 0);
|
|
843
|
+
device_define_property(this, "destroyed", false);
|
|
844
|
+
device_define_property(this, "xvfbInstance", void 0);
|
|
845
|
+
device_define_property(this, "xvfbCleanup", void 0);
|
|
846
|
+
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
847
|
+
getLibnut: ()=>device_libnut,
|
|
848
|
+
useAppleScript: ()=>this.useAppleScript,
|
|
849
|
+
sendKeyViaAppleScript,
|
|
850
|
+
runPhasedScroll,
|
|
851
|
+
debug: (message)=>debugDevice(message)
|
|
852
|
+
}));
|
|
853
|
+
device_define_property(this, "useAppleScript", void 0);
|
|
854
|
+
device_define_property(this, "uri", void 0);
|
|
855
|
+
device_define_property(this, "inputPrimitives", {
|
|
739
856
|
pointer: {
|
|
740
857
|
tap: async ({ x, y })=>{
|
|
741
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
742
858
|
const targetX = Math.round(x);
|
|
743
859
|
const targetY = Math.round(y);
|
|
744
|
-
await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
860
|
+
await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
861
|
+
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
862
|
+
await this.inputDriver.delay(CLICK_HOLD_DURATION);
|
|
863
|
+
});
|
|
748
864
|
},
|
|
749
865
|
doubleClick: async ({ x, y })=>{
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
libnut.mouseClick('left', true);
|
|
866
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
867
|
+
this.inputDriver.mouseClick('left', true);
|
|
753
868
|
},
|
|
754
869
|
rightClick: async ({ x, y })=>{
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
libnut.mouseClick('right');
|
|
870
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
871
|
+
this.inputDriver.mouseClick('right');
|
|
758
872
|
},
|
|
759
873
|
hover: async ({ x, y })=>{
|
|
760
|
-
|
|
761
|
-
await
|
|
762
|
-
await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
|
|
874
|
+
await this.inputDriver.smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
875
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
763
876
|
},
|
|
764
877
|
dragAndDrop: async (from, to)=>{
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
libnut.mouseToggle('up', 'left');
|
|
878
|
+
this.inputDriver.moveMouse(Math.round(from.x), Math.round(from.y));
|
|
879
|
+
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
880
|
+
await this.inputDriver.delay(100);
|
|
881
|
+
this.inputDriver.moveMouse(Math.round(to.x), Math.round(to.y));
|
|
882
|
+
await this.inputDriver.delay(100);
|
|
883
|
+
});
|
|
772
884
|
}
|
|
773
885
|
},
|
|
774
886
|
keyboard: {
|
|
775
887
|
typeText: async (value, opts)=>{
|
|
776
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
777
888
|
const element = opts?.target;
|
|
778
889
|
if (element) {
|
|
779
890
|
const [x, y] = element.center;
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
await
|
|
891
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
892
|
+
this.inputDriver.mouseClick('left');
|
|
893
|
+
await this.inputDriver.delay(INPUT_FOCUS_DELAY);
|
|
783
894
|
if (opts?.replace !== false) {
|
|
784
895
|
await this.selectAllAndDelete();
|
|
785
|
-
await
|
|
896
|
+
await this.inputDriver.delay(INPUT_CLEAR_DELAY);
|
|
786
897
|
}
|
|
787
898
|
}
|
|
788
899
|
await this.smartTypeString(value);
|
|
789
900
|
},
|
|
790
901
|
keyboardPress: async (keyName, opts)=>{
|
|
791
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
792
902
|
const target = opts?.target;
|
|
793
903
|
if (target) {
|
|
794
904
|
const [x, y] = target.center;
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
await
|
|
905
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
906
|
+
this.inputDriver.mouseClick('left');
|
|
907
|
+
await this.inputDriver.delay(50);
|
|
798
908
|
}
|
|
799
909
|
await this.pressKeyboardShortcut(keyName);
|
|
800
910
|
},
|
|
801
911
|
clearInput: async (target)=>{
|
|
802
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
803
912
|
if (target) {
|
|
804
913
|
const element = target;
|
|
805
914
|
const [x, y] = element.center;
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
await
|
|
915
|
+
this.inputDriver.moveMouse(Math.round(x), Math.round(y));
|
|
916
|
+
this.inputDriver.mouseClick('left');
|
|
917
|
+
await this.inputDriver.delay(100);
|
|
809
918
|
}
|
|
810
919
|
await this.selectAllAndDelete();
|
|
811
|
-
await
|
|
920
|
+
await this.inputDriver.delay(50);
|
|
812
921
|
}
|
|
813
922
|
},
|
|
814
923
|
scroll: {
|
|
@@ -1129,7 +1238,7 @@ class HelperProcessRDPBackendClient {
|
|
|
1129
1238
|
function createDefaultRDPBackendClient() {
|
|
1130
1239
|
return new HelperProcessRDPBackendClient();
|
|
1131
1240
|
}
|
|
1132
|
-
function
|
|
1241
|
+
function rdp_device_define_property(obj, key, value) {
|
|
1133
1242
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
1134
1243
|
value: value,
|
|
1135
1244
|
enumerable: true,
|
|
@@ -1288,14 +1397,14 @@ class RDPDevice {
|
|
|
1288
1397
|
}
|
|
1289
1398
|
}
|
|
1290
1399
|
constructor(options){
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1400
|
+
rdp_device_define_property(this, "interfaceType", 'rdp');
|
|
1401
|
+
rdp_device_define_property(this, "options", void 0);
|
|
1402
|
+
rdp_device_define_property(this, "backend", void 0);
|
|
1403
|
+
rdp_device_define_property(this, "connectionInfo", void 0);
|
|
1404
|
+
rdp_device_define_property(this, "destroyed", false);
|
|
1405
|
+
rdp_device_define_property(this, "cursorPosition", void 0);
|
|
1406
|
+
rdp_device_define_property(this, "uri", void 0);
|
|
1407
|
+
rdp_device_define_property(this, "inputPrimitives", {
|
|
1299
1408
|
pointer: {
|
|
1300
1409
|
tap: async ({ x, y })=>{
|
|
1301
1410
|
await this.movePointer(Math.round(x), Math.round(y), {
|
|
@@ -1635,7 +1744,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
1635
1744
|
constructor(toolsManager){
|
|
1636
1745
|
super({
|
|
1637
1746
|
name: '@midscene/computer-mcp',
|
|
1638
|
-
version: "1.8.
|
|
1747
|
+
version: "1.8.7-beta-20260527113633.0",
|
|
1639
1748
|
description: 'Control the computer desktop using natural language commands'
|
|
1640
1749
|
}, toolsManager);
|
|
1641
1750
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -71,6 +71,7 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
71
71
|
private destroyed;
|
|
72
72
|
private xvfbInstance?;
|
|
73
73
|
private xvfbCleanup?;
|
|
74
|
+
private readonly inputDriver;
|
|
74
75
|
/**
|
|
75
76
|
* On macOS, use AppleScript for keyboard operations by default
|
|
76
77
|
* to avoid focus issues with system overlays (e.g. Spotlight).
|
|
@@ -23,6 +23,7 @@ declare class ComputerDevice implements AbstractInterface {
|
|
|
23
23
|
private destroyed;
|
|
24
24
|
private xvfbInstance?;
|
|
25
25
|
private xvfbCleanup?;
|
|
26
|
+
private readonly inputDriver;
|
|
26
27
|
/**
|
|
27
28
|
* On macOS, use AppleScript for keyboard operations by default
|
|
28
29
|
* to avoid focus issues with system overlays (e.g. Spotlight).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.8.
|
|
3
|
+
"version": "1.8.7-beta-20260527113633.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -38,8 +38,8 @@
|
|
|
38
38
|
"@computer-use/libnut": "^4.2.0",
|
|
39
39
|
"clipboardy": "^4.0.0",
|
|
40
40
|
"screenshot-desktop": "^1.15.3",
|
|
41
|
-
"@midscene/core": "1.8.
|
|
42
|
-
"@midscene/shared": "1.8.
|
|
41
|
+
"@midscene/core": "1.8.7-beta-20260527113633.0",
|
|
42
|
+
"@midscene/shared": "1.8.7-beta-20260527113633.0"
|
|
43
43
|
},
|
|
44
44
|
"optionalDependencies": {
|
|
45
45
|
"node-mac-permissions": "2.5.0"
|