@midscene/computer 1.9.2 → 1.9.3-beta-20260608125147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +43 -24
- package/dist/es/index.mjs +43 -24
- package/dist/es/mcp-server.mjs +43 -24
- package/dist/lib/cli.js +43 -24
- package/dist/lib/index.js +43 -24
- package/dist/lib/mcp-server.js +43 -24
- package/dist/types/index.d.ts +1 -0
- package/dist/types/mcp-server.d.ts +1 -0
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -52,6 +52,13 @@ class ComputerInputDriver {
|
|
|
52
52
|
scrollMouse(x, y) {
|
|
53
53
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
54
|
}
|
|
55
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
56
|
+
this.assertActive('emitScrollDetents');
|
|
57
|
+
for(let i = 0; i < detents; i++){
|
|
58
|
+
this.scrollMouse(dx, dy);
|
|
59
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
55
62
|
keyTap(key, modifiers) {
|
|
56
63
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
64
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -230,13 +237,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
230
237
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
231
238
|
const INPUT_FOCUS_DELAY = 300;
|
|
232
239
|
const INPUT_CLEAR_DELAY = 150;
|
|
233
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
234
240
|
const SCROLL_STEP_DELAY = 100;
|
|
235
241
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
236
242
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
237
243
|
const EDGE_SCROLL_STEPS = 400;
|
|
238
244
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
239
245
|
const PHASED_MIN_STEPS = 10;
|
|
246
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
247
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
248
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
249
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
250
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
240
251
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
241
252
|
const EDGE_SCROLL_SPEC = {
|
|
242
253
|
scrollToTop: {
|
|
@@ -244,7 +255,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
244
255
|
key: 'home',
|
|
245
256
|
libnut: [
|
|
246
257
|
0,
|
|
247
|
-
|
|
258
|
+
1
|
|
248
259
|
]
|
|
249
260
|
},
|
|
250
261
|
scrollToBottom: {
|
|
@@ -252,14 +263,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
252
263
|
key: 'end',
|
|
253
264
|
libnut: [
|
|
254
265
|
0,
|
|
255
|
-
-
|
|
266
|
+
-1
|
|
256
267
|
]
|
|
257
268
|
},
|
|
258
269
|
scrollToLeft: {
|
|
259
270
|
direction: 'left',
|
|
260
271
|
key: 'home',
|
|
261
272
|
libnut: [
|
|
262
|
-
-
|
|
273
|
+
-1,
|
|
263
274
|
0
|
|
264
275
|
]
|
|
265
276
|
},
|
|
@@ -267,7 +278,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
267
278
|
direction: 'right',
|
|
268
279
|
key: 'end',
|
|
269
280
|
libnut: [
|
|
270
|
-
|
|
281
|
+
1,
|
|
271
282
|
0
|
|
272
283
|
]
|
|
273
284
|
}
|
|
@@ -656,7 +667,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
656
667
|
}
|
|
657
668
|
async healthCheck() {
|
|
658
669
|
console.log('[HealthCheck] Starting health check...');
|
|
659
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
670
|
+
console.log("[HealthCheck] @midscene/computer v1.9.3-beta-20260608125147.0");
|
|
660
671
|
console.log('[HealthCheck] Taking screenshot...');
|
|
661
672
|
const screenshotTimeout = 15000;
|
|
662
673
|
let timeoutId;
|
|
@@ -834,7 +845,7 @@ Original error: ${lastRawMessage}`);
|
|
|
834
845
|
});
|
|
835
846
|
this.inputDriver.sendKey(key, modifiers);
|
|
836
847
|
}
|
|
837
|
-
async
|
|
848
|
+
async moveMouseToScrollTarget(param) {
|
|
838
849
|
if (param.locate) {
|
|
839
850
|
const element = param.locate;
|
|
840
851
|
const [x, y] = element.center;
|
|
@@ -843,7 +854,19 @@ Original error: ${lastRawMessage}`);
|
|
|
843
854
|
y
|
|
844
855
|
});
|
|
845
856
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
857
|
+
return;
|
|
846
858
|
}
|
|
859
|
+
const screenSize = await this.size();
|
|
860
|
+
const point = this.toGlobalPoint({
|
|
861
|
+
x: screenSize.width / 2,
|
|
862
|
+
y: screenSize.height / 2
|
|
863
|
+
});
|
|
864
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
865
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
866
|
+
return screenSize;
|
|
867
|
+
}
|
|
868
|
+
async performScroll(param) {
|
|
869
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
847
870
|
const scrollType = param?.scrollType;
|
|
848
871
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
849
872
|
if (edgeSpec) {
|
|
@@ -853,11 +876,8 @@ Original error: ${lastRawMessage}`);
|
|
|
853
876
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
854
877
|
return;
|
|
855
878
|
}
|
|
856
|
-
const [
|
|
857
|
-
|
|
858
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
859
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
860
|
-
}
|
|
879
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
880
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
861
881
|
return;
|
|
862
882
|
}
|
|
863
883
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -865,9 +885,8 @@ Original error: ${lastRawMessage}`);
|
|
|
865
885
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
866
886
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
867
887
|
let distance = param?.distance ?? void 0;
|
|
868
|
-
let screenSize;
|
|
869
888
|
if (!distance) {
|
|
870
|
-
screenSize
|
|
889
|
+
screenSize ??= await this.size();
|
|
871
890
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
872
891
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
873
892
|
}
|
|
@@ -886,30 +905,30 @@ Original error: ${lastRawMessage}`);
|
|
|
886
905
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
887
906
|
return;
|
|
888
907
|
}
|
|
889
|
-
const
|
|
890
|
-
const
|
|
908
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
909
|
+
const directionUnit = {
|
|
891
910
|
up: [
|
|
892
911
|
0,
|
|
893
|
-
|
|
912
|
+
1
|
|
894
913
|
],
|
|
895
914
|
down: [
|
|
896
915
|
0,
|
|
897
|
-
-
|
|
916
|
+
-1
|
|
898
917
|
],
|
|
899
918
|
left: [
|
|
900
|
-
-
|
|
919
|
+
-1,
|
|
901
920
|
0
|
|
902
921
|
],
|
|
903
922
|
right: [
|
|
904
|
-
|
|
923
|
+
1,
|
|
905
924
|
0
|
|
906
925
|
]
|
|
907
926
|
};
|
|
908
|
-
const [
|
|
927
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
909
928
|
0,
|
|
910
|
-
-
|
|
929
|
+
-1
|
|
911
930
|
];
|
|
912
|
-
this.inputDriver.
|
|
931
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
913
932
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
914
933
|
return;
|
|
915
934
|
}
|
|
@@ -1912,7 +1931,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
1912
1931
|
const tools = new ComputerMidsceneTools();
|
|
1913
1932
|
runToolsCLI(tools, 'midscene-computer', {
|
|
1914
1933
|
stripPrefix: 'computer_',
|
|
1915
|
-
version: "1.9.
|
|
1934
|
+
version: "1.9.3-beta-20260608125147.0",
|
|
1916
1935
|
extraCommands: createReportCliCommands()
|
|
1917
1936
|
}).catch((e)=>{
|
|
1918
1937
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -52,6 +52,13 @@ class ComputerInputDriver {
|
|
|
52
52
|
scrollMouse(x, y) {
|
|
53
53
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
54
|
}
|
|
55
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
56
|
+
this.assertActive('emitScrollDetents');
|
|
57
|
+
for(let i = 0; i < detents; i++){
|
|
58
|
+
this.scrollMouse(dx, dy);
|
|
59
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
55
62
|
keyTap(key, modifiers) {
|
|
56
63
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
64
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -230,13 +237,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
230
237
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
231
238
|
const INPUT_FOCUS_DELAY = 300;
|
|
232
239
|
const INPUT_CLEAR_DELAY = 150;
|
|
233
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
234
240
|
const SCROLL_STEP_DELAY = 100;
|
|
235
241
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
236
242
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
237
243
|
const EDGE_SCROLL_STEPS = 400;
|
|
238
244
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
239
245
|
const PHASED_MIN_STEPS = 10;
|
|
246
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
247
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
248
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
249
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
250
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
240
251
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
241
252
|
const EDGE_SCROLL_SPEC = {
|
|
242
253
|
scrollToTop: {
|
|
@@ -244,7 +255,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
244
255
|
key: 'home',
|
|
245
256
|
libnut: [
|
|
246
257
|
0,
|
|
247
|
-
|
|
258
|
+
1
|
|
248
259
|
]
|
|
249
260
|
},
|
|
250
261
|
scrollToBottom: {
|
|
@@ -252,14 +263,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
252
263
|
key: 'end',
|
|
253
264
|
libnut: [
|
|
254
265
|
0,
|
|
255
|
-
-
|
|
266
|
+
-1
|
|
256
267
|
]
|
|
257
268
|
},
|
|
258
269
|
scrollToLeft: {
|
|
259
270
|
direction: 'left',
|
|
260
271
|
key: 'home',
|
|
261
272
|
libnut: [
|
|
262
|
-
-
|
|
273
|
+
-1,
|
|
263
274
|
0
|
|
264
275
|
]
|
|
265
276
|
},
|
|
@@ -267,7 +278,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
267
278
|
direction: 'right',
|
|
268
279
|
key: 'end',
|
|
269
280
|
libnut: [
|
|
270
|
-
|
|
281
|
+
1,
|
|
271
282
|
0
|
|
272
283
|
]
|
|
273
284
|
}
|
|
@@ -656,7 +667,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
656
667
|
}
|
|
657
668
|
async healthCheck() {
|
|
658
669
|
console.log('[HealthCheck] Starting health check...');
|
|
659
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
670
|
+
console.log("[HealthCheck] @midscene/computer v1.9.3-beta-20260608125147.0");
|
|
660
671
|
console.log('[HealthCheck] Taking screenshot...');
|
|
661
672
|
const screenshotTimeout = 15000;
|
|
662
673
|
let timeoutId;
|
|
@@ -834,7 +845,7 @@ Original error: ${lastRawMessage}`);
|
|
|
834
845
|
});
|
|
835
846
|
this.inputDriver.sendKey(key, modifiers);
|
|
836
847
|
}
|
|
837
|
-
async
|
|
848
|
+
async moveMouseToScrollTarget(param) {
|
|
838
849
|
if (param.locate) {
|
|
839
850
|
const element = param.locate;
|
|
840
851
|
const [x, y] = element.center;
|
|
@@ -843,7 +854,19 @@ Original error: ${lastRawMessage}`);
|
|
|
843
854
|
y
|
|
844
855
|
});
|
|
845
856
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
857
|
+
return;
|
|
846
858
|
}
|
|
859
|
+
const screenSize = await this.size();
|
|
860
|
+
const point = this.toGlobalPoint({
|
|
861
|
+
x: screenSize.width / 2,
|
|
862
|
+
y: screenSize.height / 2
|
|
863
|
+
});
|
|
864
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
865
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
866
|
+
return screenSize;
|
|
867
|
+
}
|
|
868
|
+
async performScroll(param) {
|
|
869
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
847
870
|
const scrollType = param?.scrollType;
|
|
848
871
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
849
872
|
if (edgeSpec) {
|
|
@@ -853,11 +876,8 @@ Original error: ${lastRawMessage}`);
|
|
|
853
876
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
854
877
|
return;
|
|
855
878
|
}
|
|
856
|
-
const [
|
|
857
|
-
|
|
858
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
859
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
860
|
-
}
|
|
879
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
880
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
861
881
|
return;
|
|
862
882
|
}
|
|
863
883
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -865,9 +885,8 @@ Original error: ${lastRawMessage}`);
|
|
|
865
885
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
866
886
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
867
887
|
let distance = param?.distance ?? void 0;
|
|
868
|
-
let screenSize;
|
|
869
888
|
if (!distance) {
|
|
870
|
-
screenSize
|
|
889
|
+
screenSize ??= await this.size();
|
|
871
890
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
872
891
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
873
892
|
}
|
|
@@ -886,30 +905,30 @@ Original error: ${lastRawMessage}`);
|
|
|
886
905
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
887
906
|
return;
|
|
888
907
|
}
|
|
889
|
-
const
|
|
890
|
-
const
|
|
908
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
909
|
+
const directionUnit = {
|
|
891
910
|
up: [
|
|
892
911
|
0,
|
|
893
|
-
|
|
912
|
+
1
|
|
894
913
|
],
|
|
895
914
|
down: [
|
|
896
915
|
0,
|
|
897
|
-
-
|
|
916
|
+
-1
|
|
898
917
|
],
|
|
899
918
|
left: [
|
|
900
|
-
-
|
|
919
|
+
-1,
|
|
901
920
|
0
|
|
902
921
|
],
|
|
903
922
|
right: [
|
|
904
|
-
|
|
923
|
+
1,
|
|
905
924
|
0
|
|
906
925
|
]
|
|
907
926
|
};
|
|
908
|
-
const [
|
|
927
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
909
928
|
0,
|
|
910
|
-
-
|
|
929
|
+
-1
|
|
911
930
|
];
|
|
912
|
-
this.inputDriver.
|
|
931
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
913
932
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
914
933
|
return;
|
|
915
934
|
}
|
|
@@ -1953,7 +1972,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
1953
1972
|
}
|
|
1954
1973
|
}
|
|
1955
1974
|
function version() {
|
|
1956
|
-
const currentVersion = "1.9.
|
|
1975
|
+
const currentVersion = "1.9.3-beta-20260608125147.0";
|
|
1957
1976
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
1958
1977
|
return currentVersion;
|
|
1959
1978
|
}
|
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -52,6 +52,13 @@ class ComputerInputDriver {
|
|
|
52
52
|
scrollMouse(x, y) {
|
|
53
53
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
54
54
|
}
|
|
55
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
56
|
+
this.assertActive('emitScrollDetents');
|
|
57
|
+
for(let i = 0; i < detents; i++){
|
|
58
|
+
this.scrollMouse(dx, dy);
|
|
59
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
55
62
|
keyTap(key, modifiers) {
|
|
56
63
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
57
64
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -230,13 +237,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
230
237
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
231
238
|
const INPUT_FOCUS_DELAY = 300;
|
|
232
239
|
const INPUT_CLEAR_DELAY = 150;
|
|
233
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
234
240
|
const SCROLL_STEP_DELAY = 100;
|
|
235
241
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
236
242
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
237
243
|
const EDGE_SCROLL_STEPS = 400;
|
|
238
244
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
239
245
|
const PHASED_MIN_STEPS = 10;
|
|
246
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
247
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
248
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
249
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
250
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
240
251
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
241
252
|
const EDGE_SCROLL_SPEC = {
|
|
242
253
|
scrollToTop: {
|
|
@@ -244,7 +255,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
244
255
|
key: 'home',
|
|
245
256
|
libnut: [
|
|
246
257
|
0,
|
|
247
|
-
|
|
258
|
+
1
|
|
248
259
|
]
|
|
249
260
|
},
|
|
250
261
|
scrollToBottom: {
|
|
@@ -252,14 +263,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
252
263
|
key: 'end',
|
|
253
264
|
libnut: [
|
|
254
265
|
0,
|
|
255
|
-
-
|
|
266
|
+
-1
|
|
256
267
|
]
|
|
257
268
|
},
|
|
258
269
|
scrollToLeft: {
|
|
259
270
|
direction: 'left',
|
|
260
271
|
key: 'home',
|
|
261
272
|
libnut: [
|
|
262
|
-
-
|
|
273
|
+
-1,
|
|
263
274
|
0
|
|
264
275
|
]
|
|
265
276
|
},
|
|
@@ -267,7 +278,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
267
278
|
direction: 'right',
|
|
268
279
|
key: 'end',
|
|
269
280
|
libnut: [
|
|
270
|
-
|
|
281
|
+
1,
|
|
271
282
|
0
|
|
272
283
|
]
|
|
273
284
|
}
|
|
@@ -656,7 +667,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
656
667
|
}
|
|
657
668
|
async healthCheck() {
|
|
658
669
|
console.log('[HealthCheck] Starting health check...');
|
|
659
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
670
|
+
console.log("[HealthCheck] @midscene/computer v1.9.3-beta-20260608125147.0");
|
|
660
671
|
console.log('[HealthCheck] Taking screenshot...');
|
|
661
672
|
const screenshotTimeout = 15000;
|
|
662
673
|
let timeoutId;
|
|
@@ -834,7 +845,7 @@ Original error: ${lastRawMessage}`);
|
|
|
834
845
|
});
|
|
835
846
|
this.inputDriver.sendKey(key, modifiers);
|
|
836
847
|
}
|
|
837
|
-
async
|
|
848
|
+
async moveMouseToScrollTarget(param) {
|
|
838
849
|
if (param.locate) {
|
|
839
850
|
const element = param.locate;
|
|
840
851
|
const [x, y] = element.center;
|
|
@@ -843,7 +854,19 @@ Original error: ${lastRawMessage}`);
|
|
|
843
854
|
y
|
|
844
855
|
});
|
|
845
856
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
857
|
+
return;
|
|
846
858
|
}
|
|
859
|
+
const screenSize = await this.size();
|
|
860
|
+
const point = this.toGlobalPoint({
|
|
861
|
+
x: screenSize.width / 2,
|
|
862
|
+
y: screenSize.height / 2
|
|
863
|
+
});
|
|
864
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
865
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
866
|
+
return screenSize;
|
|
867
|
+
}
|
|
868
|
+
async performScroll(param) {
|
|
869
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
847
870
|
const scrollType = param?.scrollType;
|
|
848
871
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
849
872
|
if (edgeSpec) {
|
|
@@ -853,11 +876,8 @@ Original error: ${lastRawMessage}`);
|
|
|
853
876
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
854
877
|
return;
|
|
855
878
|
}
|
|
856
|
-
const [
|
|
857
|
-
|
|
858
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
859
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
860
|
-
}
|
|
879
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
880
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
861
881
|
return;
|
|
862
882
|
}
|
|
863
883
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -865,9 +885,8 @@ Original error: ${lastRawMessage}`);
|
|
|
865
885
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
866
886
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
867
887
|
let distance = param?.distance ?? void 0;
|
|
868
|
-
let screenSize;
|
|
869
888
|
if (!distance) {
|
|
870
|
-
screenSize
|
|
889
|
+
screenSize ??= await this.size();
|
|
871
890
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
872
891
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
873
892
|
}
|
|
@@ -886,30 +905,30 @@ Original error: ${lastRawMessage}`);
|
|
|
886
905
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
887
906
|
return;
|
|
888
907
|
}
|
|
889
|
-
const
|
|
890
|
-
const
|
|
908
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
909
|
+
const directionUnit = {
|
|
891
910
|
up: [
|
|
892
911
|
0,
|
|
893
|
-
|
|
912
|
+
1
|
|
894
913
|
],
|
|
895
914
|
down: [
|
|
896
915
|
0,
|
|
897
|
-
-
|
|
916
|
+
-1
|
|
898
917
|
],
|
|
899
918
|
left: [
|
|
900
|
-
-
|
|
919
|
+
-1,
|
|
901
920
|
0
|
|
902
921
|
],
|
|
903
922
|
right: [
|
|
904
|
-
|
|
923
|
+
1,
|
|
905
924
|
0
|
|
906
925
|
]
|
|
907
926
|
};
|
|
908
|
-
const [
|
|
927
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
909
928
|
0,
|
|
910
|
-
-
|
|
929
|
+
-1
|
|
911
930
|
];
|
|
912
|
-
this.inputDriver.
|
|
931
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
913
932
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
914
933
|
return;
|
|
915
934
|
}
|
|
@@ -1916,7 +1935,7 @@ class ComputerMCPServer extends BaseMCPServer {
|
|
|
1916
1935
|
constructor(toolsManager){
|
|
1917
1936
|
super({
|
|
1918
1937
|
name: '@midscene/computer-mcp',
|
|
1919
|
-
version: "1.9.
|
|
1938
|
+
version: "1.9.3-beta-20260608125147.0",
|
|
1920
1939
|
description: 'Control the computer desktop using natural language commands'
|
|
1921
1940
|
}, toolsManager);
|
|
1922
1941
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -78,6 +78,13 @@ class ComputerInputDriver {
|
|
|
78
78
|
scrollMouse(x, y) {
|
|
79
79
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
80
80
|
}
|
|
81
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
82
|
+
this.assertActive('emitScrollDetents');
|
|
83
|
+
for(let i = 0; i < detents; i++){
|
|
84
|
+
this.scrollMouse(dx, dy);
|
|
85
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
81
88
|
keyTap(key, modifiers) {
|
|
82
89
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
83
90
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -256,13 +263,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
256
263
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
257
264
|
const INPUT_FOCUS_DELAY = 300;
|
|
258
265
|
const INPUT_CLEAR_DELAY = 150;
|
|
259
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
260
266
|
const SCROLL_STEP_DELAY = 100;
|
|
261
267
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
262
268
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
263
269
|
const EDGE_SCROLL_STEPS = 400;
|
|
264
270
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
265
271
|
const PHASED_MIN_STEPS = 10;
|
|
272
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
273
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
274
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
275
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
276
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
266
277
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
267
278
|
const EDGE_SCROLL_SPEC = {
|
|
268
279
|
scrollToTop: {
|
|
@@ -270,7 +281,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
270
281
|
key: 'home',
|
|
271
282
|
libnut: [
|
|
272
283
|
0,
|
|
273
|
-
|
|
284
|
+
1
|
|
274
285
|
]
|
|
275
286
|
},
|
|
276
287
|
scrollToBottom: {
|
|
@@ -278,14 +289,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
278
289
|
key: 'end',
|
|
279
290
|
libnut: [
|
|
280
291
|
0,
|
|
281
|
-
-
|
|
292
|
+
-1
|
|
282
293
|
]
|
|
283
294
|
},
|
|
284
295
|
scrollToLeft: {
|
|
285
296
|
direction: 'left',
|
|
286
297
|
key: 'home',
|
|
287
298
|
libnut: [
|
|
288
|
-
-
|
|
299
|
+
-1,
|
|
289
300
|
0
|
|
290
301
|
]
|
|
291
302
|
},
|
|
@@ -293,7 +304,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
293
304
|
direction: 'right',
|
|
294
305
|
key: 'end',
|
|
295
306
|
libnut: [
|
|
296
|
-
|
|
307
|
+
1,
|
|
297
308
|
0
|
|
298
309
|
]
|
|
299
310
|
}
|
|
@@ -682,7 +693,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
682
693
|
}
|
|
683
694
|
async healthCheck() {
|
|
684
695
|
console.log('[HealthCheck] Starting health check...');
|
|
685
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
696
|
+
console.log("[HealthCheck] @midscene/computer v1.9.3-beta-20260608125147.0");
|
|
686
697
|
console.log('[HealthCheck] Taking screenshot...');
|
|
687
698
|
const screenshotTimeout = 15000;
|
|
688
699
|
let timeoutId;
|
|
@@ -860,7 +871,7 @@ Original error: ${lastRawMessage}`);
|
|
|
860
871
|
});
|
|
861
872
|
this.inputDriver.sendKey(key, modifiers);
|
|
862
873
|
}
|
|
863
|
-
async
|
|
874
|
+
async moveMouseToScrollTarget(param) {
|
|
864
875
|
if (param.locate) {
|
|
865
876
|
const element = param.locate;
|
|
866
877
|
const [x, y] = element.center;
|
|
@@ -869,7 +880,19 @@ Original error: ${lastRawMessage}`);
|
|
|
869
880
|
y
|
|
870
881
|
});
|
|
871
882
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
883
|
+
return;
|
|
872
884
|
}
|
|
885
|
+
const screenSize = await this.size();
|
|
886
|
+
const point = this.toGlobalPoint({
|
|
887
|
+
x: screenSize.width / 2,
|
|
888
|
+
y: screenSize.height / 2
|
|
889
|
+
});
|
|
890
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
891
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
892
|
+
return screenSize;
|
|
893
|
+
}
|
|
894
|
+
async performScroll(param) {
|
|
895
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
873
896
|
const scrollType = param?.scrollType;
|
|
874
897
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
875
898
|
if (edgeSpec) {
|
|
@@ -879,11 +902,8 @@ Original error: ${lastRawMessage}`);
|
|
|
879
902
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
880
903
|
return;
|
|
881
904
|
}
|
|
882
|
-
const [
|
|
883
|
-
|
|
884
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
885
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
886
|
-
}
|
|
905
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
906
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
887
907
|
return;
|
|
888
908
|
}
|
|
889
909
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -891,9 +911,8 @@ Original error: ${lastRawMessage}`);
|
|
|
891
911
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
892
912
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
893
913
|
let distance = param?.distance ?? void 0;
|
|
894
|
-
let screenSize;
|
|
895
914
|
if (!distance) {
|
|
896
|
-
screenSize
|
|
915
|
+
screenSize ??= await this.size();
|
|
897
916
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
898
917
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
899
918
|
}
|
|
@@ -912,30 +931,30 @@ Original error: ${lastRawMessage}`);
|
|
|
912
931
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
913
932
|
return;
|
|
914
933
|
}
|
|
915
|
-
const
|
|
916
|
-
const
|
|
934
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
935
|
+
const directionUnit = {
|
|
917
936
|
up: [
|
|
918
937
|
0,
|
|
919
|
-
|
|
938
|
+
1
|
|
920
939
|
],
|
|
921
940
|
down: [
|
|
922
941
|
0,
|
|
923
|
-
-
|
|
942
|
+
-1
|
|
924
943
|
],
|
|
925
944
|
left: [
|
|
926
|
-
-
|
|
945
|
+
-1,
|
|
927
946
|
0
|
|
928
947
|
],
|
|
929
948
|
right: [
|
|
930
|
-
|
|
949
|
+
1,
|
|
931
950
|
0
|
|
932
951
|
]
|
|
933
952
|
};
|
|
934
|
-
const [
|
|
953
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
935
954
|
0,
|
|
936
|
-
-
|
|
955
|
+
-1
|
|
937
956
|
];
|
|
938
|
-
this.inputDriver.
|
|
957
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
939
958
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
940
959
|
return;
|
|
941
960
|
}
|
|
@@ -1939,7 +1958,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
1939
1958
|
const tools = new ComputerMidsceneTools();
|
|
1940
1959
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
1941
1960
|
stripPrefix: 'computer_',
|
|
1942
|
-
version: "1.9.
|
|
1961
|
+
version: "1.9.3-beta-20260608125147.0",
|
|
1943
1962
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
1944
1963
|
}).catch((e)=>{
|
|
1945
1964
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -105,6 +105,13 @@ class ComputerInputDriver {
|
|
|
105
105
|
scrollMouse(x, y) {
|
|
106
106
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
107
107
|
}
|
|
108
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
109
|
+
this.assertActive('emitScrollDetents');
|
|
110
|
+
for(let i = 0; i < detents; i++){
|
|
111
|
+
this.scrollMouse(dx, dy);
|
|
112
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
108
115
|
keyTap(key, modifiers) {
|
|
109
116
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
110
117
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -283,13 +290,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
283
290
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
284
291
|
const INPUT_FOCUS_DELAY = 300;
|
|
285
292
|
const INPUT_CLEAR_DELAY = 150;
|
|
286
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
287
293
|
const SCROLL_STEP_DELAY = 100;
|
|
288
294
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
289
295
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
290
296
|
const EDGE_SCROLL_STEPS = 400;
|
|
291
297
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
292
298
|
const PHASED_MIN_STEPS = 10;
|
|
299
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
300
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
301
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
302
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
303
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
293
304
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
294
305
|
const EDGE_SCROLL_SPEC = {
|
|
295
306
|
scrollToTop: {
|
|
@@ -297,7 +308,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
297
308
|
key: 'home',
|
|
298
309
|
libnut: [
|
|
299
310
|
0,
|
|
300
|
-
|
|
311
|
+
1
|
|
301
312
|
]
|
|
302
313
|
},
|
|
303
314
|
scrollToBottom: {
|
|
@@ -305,14 +316,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
305
316
|
key: 'end',
|
|
306
317
|
libnut: [
|
|
307
318
|
0,
|
|
308
|
-
-
|
|
319
|
+
-1
|
|
309
320
|
]
|
|
310
321
|
},
|
|
311
322
|
scrollToLeft: {
|
|
312
323
|
direction: 'left',
|
|
313
324
|
key: 'home',
|
|
314
325
|
libnut: [
|
|
315
|
-
-
|
|
326
|
+
-1,
|
|
316
327
|
0
|
|
317
328
|
]
|
|
318
329
|
},
|
|
@@ -320,7 +331,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
320
331
|
direction: 'right',
|
|
321
332
|
key: 'end',
|
|
322
333
|
libnut: [
|
|
323
|
-
|
|
334
|
+
1,
|
|
324
335
|
0
|
|
325
336
|
]
|
|
326
337
|
}
|
|
@@ -709,7 +720,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
709
720
|
}
|
|
710
721
|
async healthCheck() {
|
|
711
722
|
console.log('[HealthCheck] Starting health check...');
|
|
712
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
723
|
+
console.log("[HealthCheck] @midscene/computer v1.9.3-beta-20260608125147.0");
|
|
713
724
|
console.log('[HealthCheck] Taking screenshot...');
|
|
714
725
|
const screenshotTimeout = 15000;
|
|
715
726
|
let timeoutId;
|
|
@@ -887,7 +898,7 @@ Original error: ${lastRawMessage}`);
|
|
|
887
898
|
});
|
|
888
899
|
this.inputDriver.sendKey(key, modifiers);
|
|
889
900
|
}
|
|
890
|
-
async
|
|
901
|
+
async moveMouseToScrollTarget(param) {
|
|
891
902
|
if (param.locate) {
|
|
892
903
|
const element = param.locate;
|
|
893
904
|
const [x, y] = element.center;
|
|
@@ -896,7 +907,19 @@ Original error: ${lastRawMessage}`);
|
|
|
896
907
|
y
|
|
897
908
|
});
|
|
898
909
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
910
|
+
return;
|
|
899
911
|
}
|
|
912
|
+
const screenSize = await this.size();
|
|
913
|
+
const point = this.toGlobalPoint({
|
|
914
|
+
x: screenSize.width / 2,
|
|
915
|
+
y: screenSize.height / 2
|
|
916
|
+
});
|
|
917
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
918
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
919
|
+
return screenSize;
|
|
920
|
+
}
|
|
921
|
+
async performScroll(param) {
|
|
922
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
900
923
|
const scrollType = param?.scrollType;
|
|
901
924
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
902
925
|
if (edgeSpec) {
|
|
@@ -906,11 +929,8 @@ Original error: ${lastRawMessage}`);
|
|
|
906
929
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
907
930
|
return;
|
|
908
931
|
}
|
|
909
|
-
const [
|
|
910
|
-
|
|
911
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
912
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
913
|
-
}
|
|
932
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
933
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
914
934
|
return;
|
|
915
935
|
}
|
|
916
936
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -918,9 +938,8 @@ Original error: ${lastRawMessage}`);
|
|
|
918
938
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
919
939
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
920
940
|
let distance = param?.distance ?? void 0;
|
|
921
|
-
let screenSize;
|
|
922
941
|
if (!distance) {
|
|
923
|
-
screenSize
|
|
942
|
+
screenSize ??= await this.size();
|
|
924
943
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
925
944
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
926
945
|
}
|
|
@@ -939,30 +958,30 @@ Original error: ${lastRawMessage}`);
|
|
|
939
958
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
940
959
|
return;
|
|
941
960
|
}
|
|
942
|
-
const
|
|
943
|
-
const
|
|
961
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
962
|
+
const directionUnit = {
|
|
944
963
|
up: [
|
|
945
964
|
0,
|
|
946
|
-
|
|
965
|
+
1
|
|
947
966
|
],
|
|
948
967
|
down: [
|
|
949
968
|
0,
|
|
950
|
-
-
|
|
969
|
+
-1
|
|
951
970
|
],
|
|
952
971
|
left: [
|
|
953
|
-
-
|
|
972
|
+
-1,
|
|
954
973
|
0
|
|
955
974
|
],
|
|
956
975
|
right: [
|
|
957
|
-
|
|
976
|
+
1,
|
|
958
977
|
0
|
|
959
978
|
]
|
|
960
979
|
};
|
|
961
|
-
const [
|
|
980
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
962
981
|
0,
|
|
963
|
-
-
|
|
982
|
+
-1
|
|
964
983
|
];
|
|
965
|
-
this.inputDriver.
|
|
984
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
966
985
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
967
986
|
return;
|
|
968
987
|
}
|
|
@@ -2011,7 +2030,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2011
2030
|
}
|
|
2012
2031
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2013
2032
|
function version() {
|
|
2014
|
-
const currentVersion = "1.9.
|
|
2033
|
+
const currentVersion = "1.9.3-beta-20260608125147.0";
|
|
2015
2034
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2016
2035
|
return currentVersion;
|
|
2017
2036
|
}
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -92,6 +92,13 @@ class ComputerInputDriver {
|
|
|
92
92
|
scrollMouse(x, y) {
|
|
93
93
|
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
94
94
|
}
|
|
95
|
+
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
96
|
+
this.assertActive('emitScrollDetents');
|
|
97
|
+
for(let i = 0; i < detents; i++){
|
|
98
|
+
this.scrollMouse(dx, dy);
|
|
99
|
+
if (i < detents - 1) await this.delay(delayMs);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
95
102
|
keyTap(key, modifiers) {
|
|
96
103
|
const lib = this.getLibnutOrThrow('keyTap');
|
|
97
104
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
@@ -270,13 +277,17 @@ const CLICK_HOLD_DURATION = 100;
|
|
|
270
277
|
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
271
278
|
const INPUT_FOCUS_DELAY = 300;
|
|
272
279
|
const INPUT_CLEAR_DELAY = 150;
|
|
273
|
-
const SCROLL_REPEAT_COUNT = 10;
|
|
274
280
|
const SCROLL_STEP_DELAY = 100;
|
|
275
281
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
276
282
|
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
277
283
|
const EDGE_SCROLL_STEPS = 400;
|
|
278
284
|
const PHASED_PIXELS_PER_STEP = 30;
|
|
279
285
|
const PHASED_MIN_STEPS = 10;
|
|
286
|
+
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
287
|
+
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
288
|
+
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
289
|
+
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
290
|
+
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
280
291
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
281
292
|
const EDGE_SCROLL_SPEC = {
|
|
282
293
|
scrollToTop: {
|
|
@@ -284,7 +295,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
284
295
|
key: 'home',
|
|
285
296
|
libnut: [
|
|
286
297
|
0,
|
|
287
|
-
|
|
298
|
+
1
|
|
288
299
|
]
|
|
289
300
|
},
|
|
290
301
|
scrollToBottom: {
|
|
@@ -292,14 +303,14 @@ const EDGE_SCROLL_SPEC = {
|
|
|
292
303
|
key: 'end',
|
|
293
304
|
libnut: [
|
|
294
305
|
0,
|
|
295
|
-
-
|
|
306
|
+
-1
|
|
296
307
|
]
|
|
297
308
|
},
|
|
298
309
|
scrollToLeft: {
|
|
299
310
|
direction: 'left',
|
|
300
311
|
key: 'home',
|
|
301
312
|
libnut: [
|
|
302
|
-
-
|
|
313
|
+
-1,
|
|
303
314
|
0
|
|
304
315
|
]
|
|
305
316
|
},
|
|
@@ -307,7 +318,7 @@ const EDGE_SCROLL_SPEC = {
|
|
|
307
318
|
direction: 'right',
|
|
308
319
|
key: 'end',
|
|
309
320
|
libnut: [
|
|
310
|
-
|
|
321
|
+
1,
|
|
311
322
|
0
|
|
312
323
|
]
|
|
313
324
|
}
|
|
@@ -696,7 +707,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
696
707
|
}
|
|
697
708
|
async healthCheck() {
|
|
698
709
|
console.log('[HealthCheck] Starting health check...');
|
|
699
|
-
console.log("[HealthCheck] @midscene/computer v1.9.
|
|
710
|
+
console.log("[HealthCheck] @midscene/computer v1.9.3-beta-20260608125147.0");
|
|
700
711
|
console.log('[HealthCheck] Taking screenshot...');
|
|
701
712
|
const screenshotTimeout = 15000;
|
|
702
713
|
let timeoutId;
|
|
@@ -874,7 +885,7 @@ Original error: ${lastRawMessage}`);
|
|
|
874
885
|
});
|
|
875
886
|
this.inputDriver.sendKey(key, modifiers);
|
|
876
887
|
}
|
|
877
|
-
async
|
|
888
|
+
async moveMouseToScrollTarget(param) {
|
|
878
889
|
if (param.locate) {
|
|
879
890
|
const element = param.locate;
|
|
880
891
|
const [x, y] = element.center;
|
|
@@ -883,7 +894,19 @@ Original error: ${lastRawMessage}`);
|
|
|
883
894
|
y
|
|
884
895
|
});
|
|
885
896
|
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
897
|
+
return;
|
|
886
898
|
}
|
|
899
|
+
const screenSize = await this.size();
|
|
900
|
+
const point = this.toGlobalPoint({
|
|
901
|
+
x: screenSize.width / 2,
|
|
902
|
+
y: screenSize.height / 2
|
|
903
|
+
});
|
|
904
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
905
|
+
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
906
|
+
return screenSize;
|
|
907
|
+
}
|
|
908
|
+
async performScroll(param) {
|
|
909
|
+
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
887
910
|
const scrollType = param?.scrollType;
|
|
888
911
|
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
889
912
|
if (edgeSpec) {
|
|
@@ -893,11 +916,8 @@ Original error: ${lastRawMessage}`);
|
|
|
893
916
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
894
917
|
return;
|
|
895
918
|
}
|
|
896
|
-
const [
|
|
897
|
-
|
|
898
|
-
this.inputDriver.scrollMouse(dx, dy);
|
|
899
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
900
|
-
}
|
|
919
|
+
const [ux, uy] = edgeSpec.libnut;
|
|
920
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
901
921
|
return;
|
|
902
922
|
}
|
|
903
923
|
if ('singleAction' === scrollType || !scrollType) {
|
|
@@ -905,9 +925,8 @@ Original error: ${lastRawMessage}`);
|
|
|
905
925
|
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
906
926
|
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
907
927
|
let distance = param?.distance ?? void 0;
|
|
908
|
-
let screenSize;
|
|
909
928
|
if (!distance) {
|
|
910
|
-
screenSize
|
|
929
|
+
screenSize ??= await this.size();
|
|
911
930
|
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
912
931
|
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
913
932
|
}
|
|
@@ -926,30 +945,30 @@ Original error: ${lastRawMessage}`);
|
|
|
926
945
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
927
946
|
return;
|
|
928
947
|
}
|
|
929
|
-
const
|
|
930
|
-
const
|
|
948
|
+
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
949
|
+
const directionUnit = {
|
|
931
950
|
up: [
|
|
932
951
|
0,
|
|
933
|
-
|
|
952
|
+
1
|
|
934
953
|
],
|
|
935
954
|
down: [
|
|
936
955
|
0,
|
|
937
|
-
-
|
|
956
|
+
-1
|
|
938
957
|
],
|
|
939
958
|
left: [
|
|
940
|
-
-
|
|
959
|
+
-1,
|
|
941
960
|
0
|
|
942
961
|
],
|
|
943
962
|
right: [
|
|
944
|
-
|
|
963
|
+
1,
|
|
945
964
|
0
|
|
946
965
|
]
|
|
947
966
|
};
|
|
948
|
-
const [
|
|
967
|
+
const [ux, uy] = directionUnit[direction] || [
|
|
949
968
|
0,
|
|
950
|
-
-
|
|
969
|
+
-1
|
|
951
970
|
];
|
|
952
|
-
this.inputDriver.
|
|
971
|
+
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
953
972
|
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
954
973
|
return;
|
|
955
974
|
}
|
|
@@ -1959,7 +1978,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
1959
1978
|
constructor(toolsManager){
|
|
1960
1979
|
super({
|
|
1961
1980
|
name: '@midscene/computer-mcp',
|
|
1962
|
-
version: "1.9.
|
|
1981
|
+
version: "1.9.3-beta-20260608125147.0",
|
|
1963
1982
|
description: 'Control the computer desktop using natural language commands'
|
|
1964
1983
|
}, toolsManager);
|
|
1965
1984
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -119,6 +119,7 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
119
119
|
private smartTypeString;
|
|
120
120
|
private selectAllAndDelete;
|
|
121
121
|
private pressKeyboardShortcut;
|
|
122
|
+
private moveMouseToScrollTarget;
|
|
122
123
|
private performScroll;
|
|
123
124
|
actionSpace(): DeviceAction<any>[];
|
|
124
125
|
destroy(): Promise<void>;
|
|
@@ -71,6 +71,7 @@ declare class ComputerDevice implements AbstractInterface {
|
|
|
71
71
|
private smartTypeString;
|
|
72
72
|
private selectAllAndDelete;
|
|
73
73
|
private pressKeyboardShortcut;
|
|
74
|
+
private moveMouseToScrollTarget;
|
|
74
75
|
private performScroll;
|
|
75
76
|
actionSpace(): DeviceAction<any>[];
|
|
76
77
|
destroy(): Promise<void>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.9.
|
|
3
|
+
"version": "1.9.3-beta-20260608125147.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -38,8 +38,8 @@
|
|
|
38
38
|
"@computer-use/libnut": "^4.2.0",
|
|
39
39
|
"clipboardy": "^4.0.0",
|
|
40
40
|
"screenshot-desktop": "^1.15.3",
|
|
41
|
-
"@midscene/core": "1.9.
|
|
42
|
-
"@midscene/shared": "1.9.
|
|
41
|
+
"@midscene/core": "1.9.3-beta-20260608125147.0",
|
|
42
|
+
"@midscene/shared": "1.9.3-beta-20260608125147.0"
|
|
43
43
|
},
|
|
44
44
|
"optionalDependencies": {
|
|
45
45
|
"node-mac-permissions": "2.5.0"
|