@midscene/computer 1.9.6 → 1.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +81 -24
- package/dist/es/index.mjs +81 -24
- package/dist/es/mcp-server.mjs +81 -24
- package/dist/lib/cli.js +81 -24
- package/dist/lib/index.js +81 -24
- package/dist/lib/mcp-server.js +81 -24
- package/dist/types/index.d.ts +2 -0
- package/dist/types/mcp-server.d.ts +2 -0
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -40,6 +40,33 @@ class ComputerInputDriver {
|
|
|
40
40
|
moveMouse(x, y) {
|
|
41
41
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
42
42
|
}
|
|
43
|
+
focusActiveWindow() {
|
|
44
|
+
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
45
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
46
|
+
try {
|
|
47
|
+
const handle = lib.getActiveWindow();
|
|
48
|
+
if (!handle) return false;
|
|
49
|
+
lib.focusWindow(handle);
|
|
50
|
+
return true;
|
|
51
|
+
} catch (error) {
|
|
52
|
+
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
getActiveWindowRect() {
|
|
57
|
+
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
58
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
59
|
+
try {
|
|
60
|
+
const handle = lib.getActiveWindow();
|
|
61
|
+
if (!handle) return null;
|
|
62
|
+
const rect = lib.getWindowRect(handle);
|
|
63
|
+
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
64
|
+
return rect;
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
43
70
|
mouseClick(button, double) {
|
|
44
71
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
45
72
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -52,6 +79,13 @@ class ComputerInputDriver {
|
|
|
52
79
|
scrollMouse(x, y) {
|
|
53
80
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
81
|
}
|
|
82
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
83
|
+
this.assertActive('emitScrollDetents');
|
|
84
|
+
for(let i = 0; i < detents; i++){
|
|
85
|
+
this.scrollMouse(dx, dy);
|
|
86
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
55
89
|
keyTap(key, modifiers) {
|
|
56
90
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
91
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -230,13 +264,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
230
264
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
231
265
|
const INPUT_FOCUS_DELAY = 300;
|
|
232
266
|
const INPUT_CLEAR_DELAY = 150;
|
|
233
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
234
267
|
const SCROLL_STEP_DELAY = 100;
|
|
235
268
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
236
269
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
237
270
|
const EDGE_SCROLL_STEPS = 400;
|
|
238
271
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
239
272
|
const PHASED_MIN_STEPS = 10;
|
|
273
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
274
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
275
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
276
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
277
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
240
278
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
241
279
|
const EDGE_SCROLL_SPEC = {
|
|
242
280
|
scrollToTop: {
|
|
@@ -244,7 +282,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
244
282
|
key: 'home',
|
|
245
283
|
libnut: [
|
|
246
284
|
0,
|
|
247
|
-
|
|
285
|
+
1
|
|
248
286
|
]
|
|
249
287
|
},
|
|
250
288
|
scrollToBottom: {
|
|
@@ -252,14 +290,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
252
290
|
key: 'end',
|
|
253
291
|
libnut: [
|
|
254
292
|
0,
|
|
255
|
-
-
|
|
293
|
+
-1
|
|
256
294
|
]
|
|
257
295
|
},
|
|
258
296
|
scrollToLeft: {
|
|
259
297
|
direction: 'left',
|
|
260
298
|
key: 'home',
|
|
261
299
|
libnut: [
|
|
262
|
-
-
|
|
300
|
+
-1,
|
|
263
301
|
0
|
|
264
302
|
]
|
|
265
303
|
},
|
|
@@ -267,7 +305,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
267
305
|
direction: 'right',
|
|
268
306
|
key: 'end',
|
|
269
307
|
libnut: [
|
|
270
|
-
|
|
308
|
+
1,
|
|
271
309
|
0
|
|
272
310
|
]
|
|
273
311
|
}
|
|
@@ -656,7 +694,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
656
694
|
}
|
|
657
695
|
async healthCheck() {
|
|
658
696
|
console.log('[HealthCheck] Starting health check...');
|
|
659
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
697
|
+
console.log("[HealthCheck] @midscene/computer v1.9.7");
|
|
660
698
|
console.log('[HealthCheck] Taking screenshot...');
|
|
661
699
|
const screenshotTimeout = 15000;
|
|
662
700
|
let timeoutId;
|
|
@@ -834,7 +872,20 @@ Original error: ${lastRawMessage}`);
|
|
|
834
872
|
});
|
|
835
873
|
this.inputDriver.sendKey(key, modifiers);
|
|
836
874
|
}
|
|
837
|
-
|
|
875
|
+
resolveUntargetedScrollPoint(screenSize) {
|
|
876
|
+
if ('win32' === process.platform) {
|
|
877
|
+
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
878
|
+
if (activeWindowRect) return {
|
|
879
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
880
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
881
|
+
};
|
|
882
|
+
}
|
|
883
|
+
return this.toGlobalPoint({
|
|
884
|
+
x: screenSize.width / 2,
|
|
885
|
+
y: screenSize.height / 2
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
async moveMouseToScrollTarget(param) {
|
|
838
889
|
if (param.locate) {
|
|
839
890
|
const element = param.locate;
|
|
840
891
|
const [x, y] = element.center;
|
|
@@ -843,7 +894,17 @@ Original error: ${lastRawMessage}`);
|
|
|
843
894
|
y
|
|
844
895
|
});
|
|
845
896
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
897
|
+
return;
|
|
846
898
|
}
|
|
899
|
+
const screenSize = await this.size();
|
|
900
|
+
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
901
|
+
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
902
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
903
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
904
|
+
return screenSize;
|
|
905
|
+
}
|
|
906
|
+
async performScroll(param) {
|
|
907
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
847
908
|
const scrollType = param?.scrollType;
|
|
848
909
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
849
910
|
if (edgeSpec) {
|
|
@@ -853,11 +914,8 @@ Original error: ${lastRawMessage}`);
|
|
|
853
914
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
854
915
|
return;
|
|
855
916
|
}
|
|
856
|
-
const [
|
|
857
|
-
|
|
858
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
859
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
860
|
-
}
|
|
917
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
918
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
861
919
|
return;
|
|
862
920
|
}
|
|
863
921
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -865,9 +923,8 @@ Original error: ${lastRawMessage}`);
|
|
|
865
923
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
866
924
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
867
925
|
let distance = param?.distance ?? void 0;
|
|
868
|
-
let screenSize;
|
|
869
926
|
if (!distance) {
|
|
870
|
-
screenSize
|
|
927
|
+
screenSize ??= await this.size();
|
|
871
928
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
872
929
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
873
930
|
}
|
|
@@ -886,30 +943,30 @@ Original error: ${lastRawMessage}`);
|
|
|
886
943
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
887
944
|
return;
|
|
888
945
|
}
|
|
889
|
-
const
|
|
890
|
-
const
|
|
946
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
947
|
+
const directionUnit = {
|
|
891
948
|
up: [
|
|
892
949
|
0,
|
|
893
|
-
|
|
950
|
+
1
|
|
894
951
|
],
|
|
895
952
|
down: [
|
|
896
953
|
0,
|
|
897
|
-
-
|
|
954
|
+
-1
|
|
898
955
|
],
|
|
899
956
|
left: [
|
|
900
|
-
-
|
|
957
|
+
-1,
|
|
901
958
|
0
|
|
902
959
|
],
|
|
903
960
|
right: [
|
|
904
|
-
|
|
961
|
+
1,
|
|
905
962
|
0
|
|
906
963
|
]
|
|
907
964
|
};
|
|
908
|
-
const [
|
|
965
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
909
966
|
0,
|
|
910
|
-
-
|
|
967
|
+
-1
|
|
911
968
|
];
|
|
912
|
-
this.inputDriver.
|
|
969
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
913
970
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
914
971
|
return;
|
|
915
972
|
}
|
|
@@ -1998,7 +2055,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
1998
2055
|
const tools = new ComputerMidsceneTools();
|
|
1999
2056
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2000
2057
|
stripPrefix: 'computer_',
|
|
2001
|
-
version: "1.9.
|
|
2058
|
+
version: "1.9.7",
|
|
2002
2059
|
extraCommands: createReportCliCommands()
|
|
2003
2060
|
}).catch((e)=>{
|
|
2004
2061
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -40,6 +40,33 @@ class ComputerInputDriver {
|
|
|
40
40
|
moveMouse(x, y) {
|
|
41
41
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
42
42
|
}
|
|
43
|
+
focusActiveWindow() {
|
|
44
|
+
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
45
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
46
|
+
try {
|
|
47
|
+
const handle = lib.getActiveWindow();
|
|
48
|
+
if (!handle) return false;
|
|
49
|
+
lib.focusWindow(handle);
|
|
50
|
+
return true;
|
|
51
|
+
} catch (error) {
|
|
52
|
+
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
getActiveWindowRect() {
|
|
57
|
+
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
58
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
59
|
+
try {
|
|
60
|
+
const handle = lib.getActiveWindow();
|
|
61
|
+
if (!handle) return null;
|
|
62
|
+
const rect = lib.getWindowRect(handle);
|
|
63
|
+
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
64
|
+
return rect;
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
43
70
|
mouseClick(button, double) {
|
|
44
71
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
45
72
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -52,6 +79,13 @@ class ComputerInputDriver {
|
|
|
52
79
|
scrollMouse(x, y) {
|
|
53
80
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
81
|
}
|
|
82
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
83
|
+
this.assertActive('emitScrollDetents');
|
|
84
|
+
for(let i = 0; i < detents; i++){
|
|
85
|
+
this.scrollMouse(dx, dy);
|
|
86
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
55
89
|
keyTap(key, modifiers) {
|
|
56
90
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
91
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -230,13 +264,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
230
264
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
231
265
|
const INPUT_FOCUS_DELAY = 300;
|
|
232
266
|
const INPUT_CLEAR_DELAY = 150;
|
|
233
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
234
267
|
const SCROLL_STEP_DELAY = 100;
|
|
235
268
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
236
269
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
237
270
|
const EDGE_SCROLL_STEPS = 400;
|
|
238
271
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
239
272
|
const PHASED_MIN_STEPS = 10;
|
|
273
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
274
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
275
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
276
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
277
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
240
278
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
241
279
|
const EDGE_SCROLL_SPEC = {
|
|
242
280
|
scrollToTop: {
|
|
@@ -244,7 +282,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
244
282
|
key: 'home',
|
|
245
283
|
libnut: [
|
|
246
284
|
0,
|
|
247
|
-
|
|
285
|
+
1
|
|
248
286
|
]
|
|
249
287
|
},
|
|
250
288
|
scrollToBottom: {
|
|
@@ -252,14 +290,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
252
290
|
key: 'end',
|
|
253
291
|
libnut: [
|
|
254
292
|
0,
|
|
255
|
-
-
|
|
293
|
+
-1
|
|
256
294
|
]
|
|
257
295
|
},
|
|
258
296
|
scrollToLeft: {
|
|
259
297
|
direction: 'left',
|
|
260
298
|
key: 'home',
|
|
261
299
|
libnut: [
|
|
262
|
-
-
|
|
300
|
+
-1,
|
|
263
301
|
0
|
|
264
302
|
]
|
|
265
303
|
},
|
|
@@ -267,7 +305,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
267
305
|
direction: 'right',
|
|
268
306
|
key: 'end',
|
|
269
307
|
libnut: [
|
|
270
|
-
|
|
308
|
+
1,
|
|
271
309
|
0
|
|
272
310
|
]
|
|
273
311
|
}
|
|
@@ -656,7 +694,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
656
694
|
}
|
|
657
695
|
async healthCheck() {
|
|
658
696
|
console.log('[HealthCheck] Starting health check...');
|
|
659
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
697
|
+
console.log("[HealthCheck] @midscene/computer v1.9.7");
|
|
660
698
|
console.log('[HealthCheck] Taking screenshot...');
|
|
661
699
|
const screenshotTimeout = 15000;
|
|
662
700
|
let timeoutId;
|
|
@@ -834,7 +872,20 @@ Original error: ${lastRawMessage}`);
|
|
|
834
872
|
});
|
|
835
873
|
this.inputDriver.sendKey(key, modifiers);
|
|
836
874
|
}
|
|
837
|
-
|
|
875
|
+
resolveUntargetedScrollPoint(screenSize) {
|
|
876
|
+
if ('win32' === process.platform) {
|
|
877
|
+
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
878
|
+
if (activeWindowRect) return {
|
|
879
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
880
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
881
|
+
};
|
|
882
|
+
}
|
|
883
|
+
return this.toGlobalPoint({
|
|
884
|
+
x: screenSize.width / 2,
|
|
885
|
+
y: screenSize.height / 2
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
async moveMouseToScrollTarget(param) {
|
|
838
889
|
if (param.locate) {
|
|
839
890
|
const element = param.locate;
|
|
840
891
|
const [x, y] = element.center;
|
|
@@ -843,7 +894,17 @@ Original error: ${lastRawMessage}`);
|
|
|
843
894
|
y
|
|
844
895
|
});
|
|
845
896
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
897
|
+
return;
|
|
846
898
|
}
|
|
899
|
+
const screenSize = await this.size();
|
|
900
|
+
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
901
|
+
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
902
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
903
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
904
|
+
return screenSize;
|
|
905
|
+
}
|
|
906
|
+
async performScroll(param) {
|
|
907
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
847
908
|
const scrollType = param?.scrollType;
|
|
848
909
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
849
910
|
if (edgeSpec) {
|
|
@@ -853,11 +914,8 @@ Original error: ${lastRawMessage}`);
|
|
|
853
914
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
854
915
|
return;
|
|
855
916
|
}
|
|
856
|
-
const [
|
|
857
|
-
|
|
858
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
859
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
860
|
-
}
|
|
917
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
918
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
861
919
|
return;
|
|
862
920
|
}
|
|
863
921
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -865,9 +923,8 @@ Original error: ${lastRawMessage}`);
|
|
|
865
923
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
866
924
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
867
925
|
let distance = param?.distance ?? void 0;
|
|
868
|
-
let screenSize;
|
|
869
926
|
if (!distance) {
|
|
870
|
-
screenSize
|
|
927
|
+
screenSize ??= await this.size();
|
|
871
928
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
872
929
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
873
930
|
}
|
|
@@ -886,30 +943,30 @@ Original error: ${lastRawMessage}`);
|
|
|
886
943
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
887
944
|
return;
|
|
888
945
|
}
|
|
889
|
-
const
|
|
890
|
-
const
|
|
946
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
947
|
+
const directionUnit = {
|
|
891
948
|
up: [
|
|
892
949
|
0,
|
|
893
|
-
|
|
950
|
+
1
|
|
894
951
|
],
|
|
895
952
|
down: [
|
|
896
953
|
0,
|
|
897
|
-
-
|
|
954
|
+
-1
|
|
898
955
|
],
|
|
899
956
|
left: [
|
|
900
|
-
-
|
|
957
|
+
-1,
|
|
901
958
|
0
|
|
902
959
|
],
|
|
903
960
|
right: [
|
|
904
|
-
|
|
961
|
+
1,
|
|
905
962
|
0
|
|
906
963
|
]
|
|
907
964
|
};
|
|
908
|
-
const [
|
|
965
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
909
966
|
0,
|
|
910
|
-
-
|
|
967
|
+
-1
|
|
911
968
|
];
|
|
912
|
-
this.inputDriver.
|
|
969
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
913
970
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
914
971
|
return;
|
|
915
972
|
}
|
|
@@ -2039,7 +2096,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2039
2096
|
}
|
|
2040
2097
|
}
|
|
2041
2098
|
function version() {
|
|
2042
|
-
const currentVersion = "1.9.
|
|
2099
|
+
const currentVersion = "1.9.7";
|
|
2043
2100
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2044
2101
|
return currentVersion;
|
|
2045
2102
|
}
|
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -40,6 +40,33 @@ class ComputerInputDriver {
|
|
|
40
40
|
moveMouse(x, y) {
|
|
41
41
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
42
42
|
}
|
|
43
|
+
focusActiveWindow() {
|
|
44
|
+
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
45
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
46
|
+
try {
|
|
47
|
+
const handle = lib.getActiveWindow();
|
|
48
|
+
if (!handle) return false;
|
|
49
|
+
lib.focusWindow(handle);
|
|
50
|
+
return true;
|
|
51
|
+
} catch (error) {
|
|
52
|
+
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
getActiveWindowRect() {
|
|
57
|
+
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
58
|
+
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
59
|
+
try {
|
|
60
|
+
const handle = lib.getActiveWindow();
|
|
61
|
+
if (!handle) return null;
|
|
62
|
+
const rect = lib.getWindowRect(handle);
|
|
63
|
+
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
64
|
+
return rect;
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
43
70
|
mouseClick(button, double) {
|
|
44
71
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
45
72
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -52,6 +79,13 @@ class ComputerInputDriver {
|
|
|
52
79
|
scrollMouse(x, y) {
|
|
53
80
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
81
|
}
|
|
82
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
83
|
+
this.assertActive('emitScrollDetents');
|
|
84
|
+
for(let i = 0; i < detents; i++){
|
|
85
|
+
this.scrollMouse(dx, dy);
|
|
86
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
55
89
|
keyTap(key, modifiers) {
|
|
56
90
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
91
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -230,13 +264,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
230
264
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
231
265
|
const INPUT_FOCUS_DELAY = 300;
|
|
232
266
|
const INPUT_CLEAR_DELAY = 150;
|
|
233
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
234
267
|
const SCROLL_STEP_DELAY = 100;
|
|
235
268
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
236
269
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
237
270
|
const EDGE_SCROLL_STEPS = 400;
|
|
238
271
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
239
272
|
const PHASED_MIN_STEPS = 10;
|
|
273
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
274
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
275
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
276
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
277
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
240
278
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
241
279
|
const EDGE_SCROLL_SPEC = {
|
|
242
280
|
scrollToTop: {
|
|
@@ -244,7 +282,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
244
282
|
key: 'home',
|
|
245
283
|
libnut: [
|
|
246
284
|
0,
|
|
247
|
-
|
|
285
|
+
1
|
|
248
286
|
]
|
|
249
287
|
},
|
|
250
288
|
scrollToBottom: {
|
|
@@ -252,14 +290,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
252
290
|
key: 'end',
|
|
253
291
|
libnut: [
|
|
254
292
|
0,
|
|
255
|
-
-
|
|
293
|
+
-1
|
|
256
294
|
]
|
|
257
295
|
},
|
|
258
296
|
scrollToLeft: {
|
|
259
297
|
direction: 'left',
|
|
260
298
|
key: 'home',
|
|
261
299
|
libnut: [
|
|
262
|
-
-
|
|
300
|
+
-1,
|
|
263
301
|
0
|
|
264
302
|
]
|
|
265
303
|
},
|
|
@@ -267,7 +305,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
267
305
|
direction: 'right',
|
|
268
306
|
key: 'end',
|
|
269
307
|
libnut: [
|
|
270
|
-
|
|
308
|
+
1,
|
|
271
309
|
0
|
|
272
310
|
]
|
|
273
311
|
}
|
|
@@ -656,7 +694,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
656
694
|
}
|
|
657
695
|
async healthCheck() {
|
|
658
696
|
console.log('[HealthCheck] Starting health check...');
|
|
659
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
697
|
+
console.log("[HealthCheck] @midscene/computer v1.9.7");
|
|
660
698
|
console.log('[HealthCheck] Taking screenshot...');
|
|
661
699
|
const screenshotTimeout = 15000;
|
|
662
700
|
let timeoutId;
|
|
@@ -834,7 +872,20 @@ Original error: ${lastRawMessage}`);
|
|
|
834
872
|
});
|
|
835
873
|
this.inputDriver.sendKey(key, modifiers);
|
|
836
874
|
}
|
|
837
|
-
|
|
875
|
+
resolveUntargetedScrollPoint(screenSize) {
|
|
876
|
+
if ('win32' === process.platform) {
|
|
877
|
+
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
878
|
+
if (activeWindowRect) return {
|
|
879
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
880
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
881
|
+
};
|
|
882
|
+
}
|
|
883
|
+
return this.toGlobalPoint({
|
|
884
|
+
x: screenSize.width / 2,
|
|
885
|
+
y: screenSize.height / 2
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
async moveMouseToScrollTarget(param) {
|
|
838
889
|
if (param.locate) {
|
|
839
890
|
const element = param.locate;
|
|
840
891
|
const [x, y] = element.center;
|
|
@@ -843,7 +894,17 @@ Original error: ${lastRawMessage}`);
|
|
|
843
894
|
y
|
|
844
895
|
});
|
|
845
896
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
897
|
+
return;
|
|
846
898
|
}
|
|
899
|
+
const screenSize = await this.size();
|
|
900
|
+
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
901
|
+
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
902
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
903
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
904
|
+
return screenSize;
|
|
905
|
+
}
|
|
906
|
+
async performScroll(param) {
|
|
907
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
847
908
|
const scrollType = param?.scrollType;
|
|
848
909
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
849
910
|
if (edgeSpec) {
|
|
@@ -853,11 +914,8 @@ Original error: ${lastRawMessage}`);
|
|
|
853
914
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
854
915
|
return;
|
|
855
916
|
}
|
|
856
|
-
const [
|
|
857
|
-
|
|
858
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
859
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
860
|
-
}
|
|
917
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
918
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
861
919
|
return;
|
|
862
920
|
}
|
|
863
921
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -865,9 +923,8 @@ Original error: ${lastRawMessage}`);
|
|
|
865
923
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
866
924
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
867
925
|
let distance = param?.distance ?? void 0;
|
|
868
|
-
let screenSize;
|
|
869
926
|
if (!distance) {
|
|
870
|
-
screenSize
|
|
927
|
+
screenSize ??= await this.size();
|
|
871
928
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
872
929
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
873
930
|
}
|
|
@@ -886,30 +943,30 @@ Original error: ${lastRawMessage}`);
|
|
|
886
943
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
887
944
|
return;
|
|
888
945
|
}
|
|
889
|
-
const
|
|
890
|
-
const
|
|
946
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
947
|
+
const directionUnit = {
|
|
891
948
|
up: [
|
|
892
949
|
0,
|
|
893
|
-
|
|
950
|
+
1
|
|
894
951
|
],
|
|
895
952
|
down: [
|
|
896
953
|
0,
|
|
897
|
-
-
|
|
954
|
+
-1
|
|
898
955
|
],
|
|
899
956
|
left: [
|
|
900
|
-
-
|
|
957
|
+
-1,
|
|
901
958
|
0
|
|
902
959
|
],
|
|
903
960
|
right: [
|
|
904
|
-
|
|
961
|
+
1,
|
|
905
962
|
0
|
|
906
963
|
]
|
|
907
964
|
};
|
|
908
|
-
const [
|
|
965
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
909
966
|
0,
|
|
910
|
-
-
|
|
967
|
+
-1
|
|
911
968
|
];
|
|
912
|
-
this.inputDriver.
|
|
969
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
913
970
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
914
971
|
return;
|
|
915
972
|
}
|
|
@@ -2002,7 +2059,7 @@ class ComputerMCPServer extends BaseMCPServer {
|
|
|
2002
2059
|
constructor(toolsManager){
|
|
2003
2060
|
super({
|
|
2004
2061
|
name: '@midscene/computer-mcp',
|
|
2005
|
-
version: "1.9.
|
|
2062
|
+
version: "1.9.7",
|
|
2006
2063
|
description: 'Control the computer desktop using natural language commands'
|
|
2007
2064
|
}, toolsManager);
|
|
2008
2065
|
}
|