@midscene/computer 1.9.6 → 1.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +81 -24
- package/dist/es/index.mjs +81 -24
- package/dist/es/mcp-server.mjs +81 -24
- package/dist/lib/cli.js +81 -24
- package/dist/lib/index.js +81 -24
- package/dist/lib/mcp-server.js +81 -24
- package/dist/types/index.d.ts +2 -0
- package/dist/types/mcp-server.d.ts +2 -0
- package/package.json +3 -3
package/dist/lib/cli.js
CHANGED
|
@@ -66,6 +66,33 @@ class ComputerInputDriver {
|
|
|
66
66
|
moveMouse(x, y) {
|
|
67
67
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
68
68
|
}
|
|
69
|
+
focusActiveWindow() {
|
|
70
|
+
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
71
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
72
|
+
try {
|
|
73
|
+
const handle = lib.getActiveWindow();
|
|
74
|
+
if (!handle) return false;
|
|
75
|
+
lib.focusWindow(handle);
|
|
76
|
+
return true;
|
|
77
|
+
} catch (error) {
|
|
78
|
+
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
getActiveWindowRect() {
|
|
83
|
+
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
84
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
85
|
+
try {
|
|
86
|
+
const handle = lib.getActiveWindow();
|
|
87
|
+
if (!handle) return null;
|
|
88
|
+
const rect = lib.getWindowRect(handle);
|
|
89
|
+
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
90
|
+
return rect;
|
|
91
|
+
} catch (error) {
|
|
92
|
+
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
69
96
|
mouseClick(button, double) {
|
|
70
97
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
71
98
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -78,6 +105,13 @@ class ComputerInputDriver {
|
|
|
78
105
|
scrollMouse(x, y) {
|
|
79
106
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
80
107
|
}
|
|
108
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
109
|
+
this.assertActive('emitScrollDetents');
|
|
110
|
+
for(let i = 0; i < detents; i++){
|
|
111
|
+
this.scrollMouse(dx, dy);
|
|
112
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
81
115
|
keyTap(key, modifiers) {
|
|
82
116
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
83
117
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -256,13 +290,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
256
290
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
257
291
|
const INPUT_FOCUS_DELAY = 300;
|
|
258
292
|
const INPUT_CLEAR_DELAY = 150;
|
|
259
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
260
293
|
const SCROLL_STEP_DELAY = 100;
|
|
261
294
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
262
295
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
263
296
|
const EDGE_SCROLL_STEPS = 400;
|
|
264
297
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
265
298
|
const PHASED_MIN_STEPS = 10;
|
|
299
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
300
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
301
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
302
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
303
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
266
304
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
267
305
|
const EDGE_SCROLL_SPEC = {
|
|
268
306
|
scrollToTop: {
|
|
@@ -270,7 +308,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
270
308
|
key: 'home',
|
|
271
309
|
libnut: [
|
|
272
310
|
0,
|
|
273
|
-
|
|
311
|
+
1
|
|
274
312
|
]
|
|
275
313
|
},
|
|
276
314
|
scrollToBottom: {
|
|
@@ -278,14 +316,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
278
316
|
key: 'end',
|
|
279
317
|
libnut: [
|
|
280
318
|
0,
|
|
281
|
-
-
|
|
319
|
+
-1
|
|
282
320
|
]
|
|
283
321
|
},
|
|
284
322
|
scrollToLeft: {
|
|
285
323
|
direction: 'left',
|
|
286
324
|
key: 'home',
|
|
287
325
|
libnut: [
|
|
288
|
-
-
|
|
326
|
+
-1,
|
|
289
327
|
0
|
|
290
328
|
]
|
|
291
329
|
},
|
|
@@ -293,7 +331,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
293
331
|
direction: 'right',
|
|
294
332
|
key: 'end',
|
|
295
333
|
libnut: [
|
|
296
|
-
|
|
334
|
+
1,
|
|
297
335
|
0
|
|
298
336
|
]
|
|
299
337
|
}
|
|
@@ -682,7 +720,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
682
720
|
}
|
|
683
721
|
async healthCheck() {
|
|
684
722
|
console.log('[HealthCheck] Starting health check...');
|
|
685
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
723
|
+
console.log("[HealthCheck] @midscene/computer v1.9.7");
|
|
686
724
|
console.log('[HealthCheck] Taking screenshot...');
|
|
687
725
|
const screenshotTimeout = 15000;
|
|
688
726
|
let timeoutId;
|
|
@@ -860,7 +898,20 @@ Original error: ${lastRawMessage}`);
|
|
|
860
898
|
});
|
|
861
899
|
this.inputDriver.sendKey(key, modifiers);
|
|
862
900
|
}
|
|
863
|
-
|
|
901
|
+
resolveUntargetedScrollPoint(screenSize) {
|
|
902
|
+
if ('win32' === process.platform) {
|
|
903
|
+
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
904
|
+
if (activeWindowRect) return {
|
|
905
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
906
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
return this.toGlobalPoint({
|
|
910
|
+
x: screenSize.width / 2,
|
|
911
|
+
y: screenSize.height / 2
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
async moveMouseToScrollTarget(param) {
|
|
864
915
|
if (param.locate) {
|
|
865
916
|
const element = param.locate;
|
|
866
917
|
const [x, y] = element.center;
|
|
@@ -869,7 +920,17 @@ Original error: ${lastRawMessage}`);
|
|
|
869
920
|
y
|
|
870
921
|
});
|
|
871
922
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
923
|
+
return;
|
|
872
924
|
}
|
|
925
|
+
const screenSize = await this.size();
|
|
926
|
+
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
927
|
+
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
928
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
929
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
930
|
+
return screenSize;
|
|
931
|
+
}
|
|
932
|
+
async performScroll(param) {
|
|
933
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
873
934
|
const scrollType = param?.scrollType;
|
|
874
935
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
875
936
|
if (edgeSpec) {
|
|
@@ -879,11 +940,8 @@ Original error: ${lastRawMessage}`);
|
|
|
879
940
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
880
941
|
return;
|
|
881
942
|
}
|
|
882
|
-
const [
|
|
883
|
-
|
|
884
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
885
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
886
|
-
}
|
|
943
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
944
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
887
945
|
return;
|
|
888
946
|
}
|
|
889
947
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -891,9 +949,8 @@ Original error: ${lastRawMessage}`);
|
|
|
891
949
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
892
950
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
893
951
|
let distance = param?.distance ?? void 0;
|
|
894
|
-
let screenSize;
|
|
895
952
|
if (!distance) {
|
|
896
|
-
screenSize
|
|
953
|
+
screenSize ??= await this.size();
|
|
897
954
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
898
955
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
899
956
|
}
|
|
@@ -912,30 +969,30 @@ Original error: ${lastRawMessage}`);
|
|
|
912
969
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
913
970
|
return;
|
|
914
971
|
}
|
|
915
|
-
const
|
|
916
|
-
const
|
|
972
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
973
|
+
const directionUnit = {
|
|
917
974
|
up: [
|
|
918
975
|
0,
|
|
919
|
-
|
|
976
|
+
1
|
|
920
977
|
],
|
|
921
978
|
down: [
|
|
922
979
|
0,
|
|
923
|
-
-
|
|
980
|
+
-1
|
|
924
981
|
],
|
|
925
982
|
left: [
|
|
926
|
-
-
|
|
983
|
+
-1,
|
|
927
984
|
0
|
|
928
985
|
],
|
|
929
986
|
right: [
|
|
930
|
-
|
|
987
|
+
1,
|
|
931
988
|
0
|
|
932
989
|
]
|
|
933
990
|
};
|
|
934
|
-
const [
|
|
991
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
935
992
|
0,
|
|
936
|
-
-
|
|
993
|
+
-1
|
|
937
994
|
];
|
|
938
|
-
this.inputDriver.
|
|
995
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
939
996
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
940
997
|
return;
|
|
941
998
|
}
|
|
@@ -2025,7 +2082,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2025
2082
|
const tools = new ComputerMidsceneTools();
|
|
2026
2083
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2027
2084
|
stripPrefix: 'computer_',
|
|
2028
|
-
version: "1.9.
|
|
2085
|
+
version: "1.9.7",
|
|
2029
2086
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2030
2087
|
}).catch((e)=>{
|
|
2031
2088
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -93,6 +93,33 @@ class ComputerInputDriver {
|
|
|
93
93
|
moveMouse(x, y) {
|
|
94
94
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
95
95
|
}
|
|
96
|
+
focusActiveWindow() {
|
|
97
|
+
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
98
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
99
|
+
try {
|
|
100
|
+
const handle = lib.getActiveWindow();
|
|
101
|
+
if (!handle) return false;
|
|
102
|
+
lib.focusWindow(handle);
|
|
103
|
+
return true;
|
|
104
|
+
} catch (error) {
|
|
105
|
+
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
getActiveWindowRect() {
|
|
110
|
+
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
111
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
112
|
+
try {
|
|
113
|
+
const handle = lib.getActiveWindow();
|
|
114
|
+
if (!handle) return null;
|
|
115
|
+
const rect = lib.getWindowRect(handle);
|
|
116
|
+
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
117
|
+
return rect;
|
|
118
|
+
} catch (error) {
|
|
119
|
+
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
96
123
|
mouseClick(button, double) {
|
|
97
124
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
98
125
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -105,6 +132,13 @@ class ComputerInputDriver {
|
|
|
105
132
|
scrollMouse(x, y) {
|
|
106
133
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
107
134
|
}
|
|
135
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
136
|
+
this.assertActive('emitScrollDetents');
|
|
137
|
+
for(let i = 0; i < detents; i++){
|
|
138
|
+
this.scrollMouse(dx, dy);
|
|
139
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
108
142
|
keyTap(key, modifiers) {
|
|
109
143
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
110
144
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -283,13 +317,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
283
317
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
284
318
|
const INPUT_FOCUS_DELAY = 300;
|
|
285
319
|
const INPUT_CLEAR_DELAY = 150;
|
|
286
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
287
320
|
const SCROLL_STEP_DELAY = 100;
|
|
288
321
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
289
322
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
290
323
|
const EDGE_SCROLL_STEPS = 400;
|
|
291
324
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
292
325
|
const PHASED_MIN_STEPS = 10;
|
|
326
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
327
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
328
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
329
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
330
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
293
331
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
294
332
|
const EDGE_SCROLL_SPEC = {
|
|
295
333
|
scrollToTop: {
|
|
@@ -297,7 +335,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
297
335
|
key: 'home',
|
|
298
336
|
libnut: [
|
|
299
337
|
0,
|
|
300
|
-
|
|
338
|
+
1
|
|
301
339
|
]
|
|
302
340
|
},
|
|
303
341
|
scrollToBottom: {
|
|
@@ -305,14 +343,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
305
343
|
key: 'end',
|
|
306
344
|
libnut: [
|
|
307
345
|
0,
|
|
308
|
-
-
|
|
346
|
+
-1
|
|
309
347
|
]
|
|
310
348
|
},
|
|
311
349
|
scrollToLeft: {
|
|
312
350
|
direction: 'left',
|
|
313
351
|
key: 'home',
|
|
314
352
|
libnut: [
|
|
315
|
-
-
|
|
353
|
+
-1,
|
|
316
354
|
0
|
|
317
355
|
]
|
|
318
356
|
},
|
|
@@ -320,7 +358,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
320
358
|
direction: 'right',
|
|
321
359
|
key: 'end',
|
|
322
360
|
libnut: [
|
|
323
|
-
|
|
361
|
+
1,
|
|
324
362
|
0
|
|
325
363
|
]
|
|
326
364
|
}
|
|
@@ -709,7 +747,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
709
747
|
}
|
|
710
748
|
async healthCheck() {
|
|
711
749
|
console.log('[HealthCheck] Starting health check...');
|
|
712
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
750
|
+
console.log("[HealthCheck] @midscene/computer v1.9.7");
|
|
713
751
|
console.log('[HealthCheck] Taking screenshot...');
|
|
714
752
|
const screenshotTimeout = 15000;
|
|
715
753
|
let timeoutId;
|
|
@@ -887,7 +925,20 @@ Original error: ${lastRawMessage}`);
|
|
|
887
925
|
});
|
|
888
926
|
this.inputDriver.sendKey(key, modifiers);
|
|
889
927
|
}
|
|
890
|
-
|
|
928
|
+
resolveUntargetedScrollPoint(screenSize) {
|
|
929
|
+
if ('win32' === process.platform) {
|
|
930
|
+
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
931
|
+
if (activeWindowRect) return {
|
|
932
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
933
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
return this.toGlobalPoint({
|
|
937
|
+
x: screenSize.width / 2,
|
|
938
|
+
y: screenSize.height / 2
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
async moveMouseToScrollTarget(param) {
|
|
891
942
|
if (param.locate) {
|
|
892
943
|
const element = param.locate;
|
|
893
944
|
const [x, y] = element.center;
|
|
@@ -896,7 +947,17 @@ Original error: ${lastRawMessage}`);
|
|
|
896
947
|
y
|
|
897
948
|
});
|
|
898
949
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
950
|
+
return;
|
|
899
951
|
}
|
|
952
|
+
const screenSize = await this.size();
|
|
953
|
+
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
954
|
+
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
955
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
956
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
957
|
+
return screenSize;
|
|
958
|
+
}
|
|
959
|
+
async performScroll(param) {
|
|
960
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
900
961
|
const scrollType = param?.scrollType;
|
|
901
962
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
902
963
|
if (edgeSpec) {
|
|
@@ -906,11 +967,8 @@ Original error: ${lastRawMessage}`);
|
|
|
906
967
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
907
968
|
return;
|
|
908
969
|
}
|
|
909
|
-
const [
|
|
910
|
-
|
|
911
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
912
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
913
|
-
}
|
|
970
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
971
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
914
972
|
return;
|
|
915
973
|
}
|
|
916
974
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -918,9 +976,8 @@ Original error: ${lastRawMessage}`);
|
|
|
918
976
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
919
977
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
920
978
|
let distance = param?.distance ?? void 0;
|
|
921
|
-
let screenSize;
|
|
922
979
|
if (!distance) {
|
|
923
|
-
screenSize
|
|
980
|
+
screenSize ??= await this.size();
|
|
924
981
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
925
982
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
926
983
|
}
|
|
@@ -939,30 +996,30 @@ Original error: ${lastRawMessage}`);
|
|
|
939
996
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
940
997
|
return;
|
|
941
998
|
}
|
|
942
|
-
const
|
|
943
|
-
const
|
|
999
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
1000
|
+
const directionUnit = {
|
|
944
1001
|
up: [
|
|
945
1002
|
0,
|
|
946
|
-
|
|
1003
|
+
1
|
|
947
1004
|
],
|
|
948
1005
|
down: [
|
|
949
1006
|
0,
|
|
950
|
-
-
|
|
1007
|
+
-1
|
|
951
1008
|
],
|
|
952
1009
|
left: [
|
|
953
|
-
-
|
|
1010
|
+
-1,
|
|
954
1011
|
0
|
|
955
1012
|
],
|
|
956
1013
|
right: [
|
|
957
|
-
|
|
1014
|
+
1,
|
|
958
1015
|
0
|
|
959
1016
|
]
|
|
960
1017
|
};
|
|
961
|
-
const [
|
|
1018
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
962
1019
|
0,
|
|
963
|
-
-
|
|
1020
|
+
-1
|
|
964
1021
|
];
|
|
965
|
-
this.inputDriver.
|
|
1022
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
966
1023
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
967
1024
|
return;
|
|
968
1025
|
}
|
|
@@ -2097,7 +2154,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2097
2154
|
}
|
|
2098
2155
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2099
2156
|
function version() {
|
|
2100
|
-
const currentVersion = "1.9.
|
|
2157
|
+
const currentVersion = "1.9.7";
|
|
2101
2158
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2102
2159
|
return currentVersion;
|
|
2103
2160
|
}
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -80,6 +80,33 @@ class ComputerInputDriver {
|
|
|
80
80
|
moveMouse(x, y) {
|
|
81
81
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
82
82
|
}
|
|
83
|
+
focusActiveWindow() {
|
|
84
|
+
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
85
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
86
|
+
try {
|
|
87
|
+
const handle = lib.getActiveWindow();
|
|
88
|
+
if (!handle) return false;
|
|
89
|
+
lib.focusWindow(handle);
|
|
90
|
+
return true;
|
|
91
|
+
} catch (error) {
|
|
92
|
+
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
getActiveWindowRect() {
|
|
97
|
+
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
98
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
99
|
+
try {
|
|
100
|
+
const handle = lib.getActiveWindow();
|
|
101
|
+
if (!handle) return null;
|
|
102
|
+
const rect = lib.getWindowRect(handle);
|
|
103
|
+
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
104
|
+
return rect;
|
|
105
|
+
} catch (error) {
|
|
106
|
+
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
83
110
|
mouseClick(button, double) {
|
|
84
111
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
85
112
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -92,6 +119,13 @@ class ComputerInputDriver {
|
|
|
92
119
|
scrollMouse(x, y) {
|
|
93
120
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
94
121
|
}
|
|
122
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
123
|
+
this.assertActive('emitScrollDetents');
|
|
124
|
+
for(let i = 0; i < detents; i++){
|
|
125
|
+
this.scrollMouse(dx, dy);
|
|
126
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
95
129
|
keyTap(key, modifiers) {
|
|
96
130
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
97
131
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -270,13 +304,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
270
304
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
271
305
|
const INPUT_FOCUS_DELAY = 300;
|
|
272
306
|
const INPUT_CLEAR_DELAY = 150;
|
|
273
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
274
307
|
const SCROLL_STEP_DELAY = 100;
|
|
275
308
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
276
309
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
277
310
|
const EDGE_SCROLL_STEPS = 400;
|
|
278
311
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
279
312
|
const PHASED_MIN_STEPS = 10;
|
|
313
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
314
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
315
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
316
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
317
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
280
318
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
281
319
|
const EDGE_SCROLL_SPEC = {
|
|
282
320
|
scrollToTop: {
|
|
@@ -284,7 +322,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
284
322
|
key: 'home',
|
|
285
323
|
libnut: [
|
|
286
324
|
0,
|
|
287
|
-
|
|
325
|
+
1
|
|
288
326
|
]
|
|
289
327
|
},
|
|
290
328
|
scrollToBottom: {
|
|
@@ -292,14 +330,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
292
330
|
key: 'end',
|
|
293
331
|
libnut: [
|
|
294
332
|
0,
|
|
295
|
-
-
|
|
333
|
+
-1
|
|
296
334
|
]
|
|
297
335
|
},
|
|
298
336
|
scrollToLeft: {
|
|
299
337
|
direction: 'left',
|
|
300
338
|
key: 'home',
|
|
301
339
|
libnut: [
|
|
302
|
-
-
|
|
340
|
+
-1,
|
|
303
341
|
0
|
|
304
342
|
]
|
|
305
343
|
},
|
|
@@ -307,7 +345,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
307
345
|
direction: 'right',
|
|
308
346
|
key: 'end',
|
|
309
347
|
libnut: [
|
|
310
|
-
|
|
348
|
+
1,
|
|
311
349
|
0
|
|
312
350
|
]
|
|
313
351
|
}
|
|
@@ -696,7 +734,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
696
734
|
}
|
|
697
735
|
async healthCheck() {
|
|
698
736
|
console.log('[HealthCheck] Starting health check...');
|
|
699
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
737
|
+
console.log("[HealthCheck] @midscene/computer v1.9.7");
|
|
700
738
|
console.log('[HealthCheck] Taking screenshot...');
|
|
701
739
|
const screenshotTimeout = 15000;
|
|
702
740
|
let timeoutId;
|
|
@@ -874,7 +912,20 @@ Original error: ${lastRawMessage}`);
|
|
|
874
912
|
});
|
|
875
913
|
this.inputDriver.sendKey(key, modifiers);
|
|
876
914
|
}
|
|
877
|
-
|
|
915
|
+
resolveUntargetedScrollPoint(screenSize) {
|
|
916
|
+
if ('win32' === process.platform) {
|
|
917
|
+
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
918
|
+
if (activeWindowRect) return {
|
|
919
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
920
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
return this.toGlobalPoint({
|
|
924
|
+
x: screenSize.width / 2,
|
|
925
|
+
y: screenSize.height / 2
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
async moveMouseToScrollTarget(param) {
|
|
878
929
|
if (param.locate) {
|
|
879
930
|
const element = param.locate;
|
|
880
931
|
const [x, y] = element.center;
|
|
@@ -883,7 +934,17 @@ Original error: ${lastRawMessage}`);
|
|
|
883
934
|
y
|
|
884
935
|
});
|
|
885
936
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
937
|
+
return;
|
|
886
938
|
}
|
|
939
|
+
const screenSize = await this.size();
|
|
940
|
+
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
941
|
+
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
942
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
943
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
944
|
+
return screenSize;
|
|
945
|
+
}
|
|
946
|
+
async performScroll(param) {
|
|
947
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
887
948
|
const scrollType = param?.scrollType;
|
|
888
949
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
889
950
|
if (edgeSpec) {
|
|
@@ -893,11 +954,8 @@ Original error: ${lastRawMessage}`);
|
|
|
893
954
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
894
955
|
return;
|
|
895
956
|
}
|
|
896
|
-
const [
|
|
897
|
-
|
|
898
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
899
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
900
|
-
}
|
|
957
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
958
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
901
959
|
return;
|
|
902
960
|
}
|
|
903
961
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -905,9 +963,8 @@ Original error: ${lastRawMessage}`);
|
|
|
905
963
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
906
964
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
907
965
|
let distance = param?.distance ?? void 0;
|
|
908
|
-
let screenSize;
|
|
909
966
|
if (!distance) {
|
|
910
|
-
screenSize
|
|
967
|
+
screenSize ??= await this.size();
|
|
911
968
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
912
969
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
913
970
|
}
|
|
@@ -926,30 +983,30 @@ Original error: ${lastRawMessage}`);
|
|
|
926
983
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
927
984
|
return;
|
|
928
985
|
}
|
|
929
|
-
const
|
|
930
|
-
const
|
|
986
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
987
|
+
const directionUnit = {
|
|
931
988
|
up: [
|
|
932
989
|
0,
|
|
933
|
-
|
|
990
|
+
1
|
|
934
991
|
],
|
|
935
992
|
down: [
|
|
936
993
|
0,
|
|
937
|
-
-
|
|
994
|
+
-1
|
|
938
995
|
],
|
|
939
996
|
left: [
|
|
940
|
-
-
|
|
997
|
+
-1,
|
|
941
998
|
0
|
|
942
999
|
],
|
|
943
1000
|
right: [
|
|
944
|
-
|
|
1001
|
+
1,
|
|
945
1002
|
0
|
|
946
1003
|
]
|
|
947
1004
|
};
|
|
948
|
-
const [
|
|
1005
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
949
1006
|
0,
|
|
950
|
-
-
|
|
1007
|
+
-1
|
|
951
1008
|
];
|
|
952
|
-
this.inputDriver.
|
|
1009
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
953
1010
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
954
1011
|
return;
|
|
955
1012
|
}
|
|
@@ -2045,7 +2102,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
2045
2102
|
constructor(toolsManager){
|
|
2046
2103
|
super({
|
|
2047
2104
|
name: '@midscene/computer-mcp',
|
|
2048
|
-
version: "1.9.
|
|
2105
|
+
version: "1.9.7",
|
|
2049
2106
|
description: 'Control the computer desktop using natural language commands'
|
|
2050
2107
|
}, toolsManager);
|
|
2051
2108
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -119,6 +119,8 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
119
119
|
private smartTypeString;
|
|
120
120
|
private selectAllAndDelete;
|
|
121
121
|
private pressKeyboardShortcut;
|
|
122
|
+
private resolveUntargetedScrollPoint;
|
|
123
|
+
private moveMouseToScrollTarget;
|
|
122
124
|
private performScroll;
|
|
123
125
|
actionSpace(): DeviceAction<any>[];
|
|
124
126
|
destroy(): Promise<void>;
|
|
@@ -71,6 +71,8 @@ declare class ComputerDevice implements AbstractInterface {
|
|
|
71
71
|
private smartTypeString;
|
|
72
72
|
private selectAllAndDelete;
|
|
73
73
|
private pressKeyboardShortcut;
|
|
74
|
+
private resolveUntargetedScrollPoint;
|
|
75
|
+
private moveMouseToScrollTarget;
|
|
74
76
|
private performScroll;
|
|
75
77
|
actionSpace(): DeviceAction<any>[];
|
|
76
78
|
destroy(): Promise<void>;
|