@midscene/computer 1.7.3 → 1.7.5-beta-20260418223706.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/darwin/phased-scroll +0 -0
- package/dist/es/cli.mjs +133 -26
- package/dist/es/index.mjs +133 -26
- package/dist/es/mcp-server.mjs +133 -26
- package/dist/lib/cli.js +132 -25
- package/dist/lib/index.js +132 -25
- package/dist/lib/mcp-server.js +132 -25
- package/package.json +4 -3
|
Binary file
|
package/dist/es/cli.mjs
CHANGED
|
@@ -4,13 +4,15 @@ import { getDebug } from "@midscene/shared/logger";
|
|
|
4
4
|
import { BaseMidsceneTools } from "@midscene/shared/mcp";
|
|
5
5
|
import { Agent } from "@midscene/core/agent";
|
|
6
6
|
import node_assert from "node:assert";
|
|
7
|
-
import { execFileSync, execSync, spawn } from "node:child_process";
|
|
7
|
+
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
8
|
+
import { existsSync } from "node:fs";
|
|
8
9
|
import { createRequire } from "node:module";
|
|
10
|
+
import { dirname, resolve as external_node_path_resolve } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
9
12
|
import { actionHoverParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
|
|
10
13
|
import { sleep } from "@midscene/core/utils";
|
|
11
14
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
12
15
|
import screenshot_desktop from "screenshot-desktop";
|
|
13
|
-
import { existsSync } from "node:fs";
|
|
14
16
|
const debugXvfb = getDebug('computer:xvfb');
|
|
15
17
|
function checkXvfbInstalled() {
|
|
16
18
|
try {
|
|
@@ -108,6 +110,45 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
108
110
|
const SCROLL_REPEAT_COUNT = 10;
|
|
109
111
|
const SCROLL_STEP_DELAY = 100;
|
|
110
112
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
113
|
+
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
114
|
+
const EDGE_SCROLL_STEPS = 400;
|
|
115
|
+
const PHASED_PIXELS_PER_STEP = 30;
|
|
116
|
+
const PHASED_MIN_STEPS = 10;
|
|
117
|
+
const APPROX_VIEWPORT_HEIGHT_PX = 600;
|
|
118
|
+
const EDGE_SCROLL_SPEC = {
|
|
119
|
+
scrollToTop: {
|
|
120
|
+
direction: 'up',
|
|
121
|
+
key: 'home',
|
|
122
|
+
libnut: [
|
|
123
|
+
0,
|
|
124
|
+
10
|
|
125
|
+
]
|
|
126
|
+
},
|
|
127
|
+
scrollToBottom: {
|
|
128
|
+
direction: 'down',
|
|
129
|
+
key: 'end',
|
|
130
|
+
libnut: [
|
|
131
|
+
0,
|
|
132
|
+
-10
|
|
133
|
+
]
|
|
134
|
+
},
|
|
135
|
+
scrollToLeft: {
|
|
136
|
+
direction: 'left',
|
|
137
|
+
key: 'home',
|
|
138
|
+
libnut: [
|
|
139
|
+
-10,
|
|
140
|
+
0
|
|
141
|
+
]
|
|
142
|
+
},
|
|
143
|
+
scrollToRight: {
|
|
144
|
+
direction: 'right',
|
|
145
|
+
key: 'end',
|
|
146
|
+
libnut: [
|
|
147
|
+
10,
|
|
148
|
+
0
|
|
149
|
+
]
|
|
150
|
+
}
|
|
151
|
+
};
|
|
111
152
|
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
112
153
|
return: 36,
|
|
113
154
|
enter: 36,
|
|
@@ -186,6 +227,69 @@ async function getLibnut() {
|
|
|
186
227
|
}
|
|
187
228
|
}
|
|
188
229
|
const debugDevice = getDebug('computer:device');
|
|
230
|
+
let phasedScrollBinaryPath;
|
|
231
|
+
function getPhasedScrollBinary() {
|
|
232
|
+
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
233
|
+
if ('darwin' !== process.platform) {
|
|
234
|
+
phasedScrollBinaryPath = null;
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
const require = createRequire(import.meta.url);
|
|
238
|
+
let pkgRoot = null;
|
|
239
|
+
try {
|
|
240
|
+
pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
|
|
241
|
+
} catch {
|
|
242
|
+
const hereDir = dirname(fileURLToPath(import.meta.url));
|
|
243
|
+
for (const candidate of [
|
|
244
|
+
external_node_path_resolve(hereDir, '..'),
|
|
245
|
+
external_node_path_resolve(hereDir, '../..')
|
|
246
|
+
])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
|
|
247
|
+
pkgRoot = candidate;
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (!pkgRoot) {
|
|
252
|
+
debugDevice('phased-scroll: cannot locate @midscene/computer package root');
|
|
253
|
+
phasedScrollBinaryPath = null;
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
|
|
257
|
+
if (!existsSync(binPath)) {
|
|
258
|
+
debugDevice('phased-scroll binary not found at', binPath);
|
|
259
|
+
phasedScrollBinaryPath = null;
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
phasedScrollBinaryPath = binPath;
|
|
263
|
+
return binPath;
|
|
264
|
+
}
|
|
265
|
+
let phasedScrollExecWarned = false;
|
|
266
|
+
function runPhasedScroll(direction, pixels, steps) {
|
|
267
|
+
const bin = getPhasedScrollBinary();
|
|
268
|
+
if (!bin) return false;
|
|
269
|
+
try {
|
|
270
|
+
const res = spawnSync(bin, [
|
|
271
|
+
direction,
|
|
272
|
+
String(Math.max(1, Math.round(pixels))),
|
|
273
|
+
String(steps)
|
|
274
|
+
], {
|
|
275
|
+
stdio: 'ignore'
|
|
276
|
+
});
|
|
277
|
+
if (0 === res.status) return true;
|
|
278
|
+
if (!phasedScrollExecWarned) {
|
|
279
|
+
phasedScrollExecWarned = true;
|
|
280
|
+
console.warn(`[@midscene/computer] phased-scroll helper exited with status ${res.status}; falling back to keyboard/libnut. This usually means Accessibility permission has not been granted to the host process.`);
|
|
281
|
+
}
|
|
282
|
+
debugDevice('phased-scroll exited non-zero', res.status, res.error);
|
|
283
|
+
return false;
|
|
284
|
+
} catch (err) {
|
|
285
|
+
if (!phasedScrollExecWarned) {
|
|
286
|
+
phasedScrollExecWarned = true;
|
|
287
|
+
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
288
|
+
}
|
|
289
|
+
debugDevice('phased-scroll spawn failed', err);
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
189
293
|
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
190
294
|
node_assert(libnut, 'libnut not initialized');
|
|
191
295
|
const currentPos = libnut.getMousePos();
|
|
@@ -305,7 +409,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
305
409
|
}
|
|
306
410
|
async healthCheck() {
|
|
307
411
|
console.log('[HealthCheck] Starting health check...');
|
|
308
|
-
console.log("[HealthCheck] @midscene/computer v1.7.
|
|
412
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260418223706.0");
|
|
309
413
|
console.log('[HealthCheck] Taking screenshot...');
|
|
310
414
|
const screenshotTimeout = 15000;
|
|
311
415
|
let timeoutId;
|
|
@@ -532,27 +636,15 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
532
636
|
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
533
637
|
}
|
|
534
638
|
const scrollType = param?.scrollType;
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
]
|
|
544
|
-
scrollToLeft: [
|
|
545
|
-
-10,
|
|
546
|
-
0
|
|
547
|
-
],
|
|
548
|
-
scrollToRight: [
|
|
549
|
-
10,
|
|
550
|
-
0
|
|
551
|
-
]
|
|
552
|
-
};
|
|
553
|
-
const edgeAction = scrollToEdgeActions[scrollType || ''];
|
|
554
|
-
if (edgeAction) {
|
|
555
|
-
const [dx, dy] = edgeAction;
|
|
639
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
640
|
+
if (edgeSpec) {
|
|
641
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
|
|
642
|
+
if (this.useAppleScript) {
|
|
643
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
644
|
+
await sleep(SCROLL_COMPLETE_DELAY);
|
|
645
|
+
return;
|
|
646
|
+
}
|
|
647
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
556
648
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
557
649
|
libnut.scrollMouse(dx, dy);
|
|
558
650
|
await sleep(SCROLL_STEP_DELAY);
|
|
@@ -561,8 +653,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
561
653
|
}
|
|
562
654
|
if ('singleAction' === scrollType || !scrollType) {
|
|
563
655
|
const distance = param?.distance || 500;
|
|
564
|
-
const ticks = Math.ceil(distance / 100);
|
|
565
656
|
const direction = param?.direction || 'down';
|
|
657
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
658
|
+
if (isKnownDirection) {
|
|
659
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
660
|
+
if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
|
|
661
|
+
}
|
|
662
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
663
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
664
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
665
|
+
for(let i = 0; i < pages; i++){
|
|
666
|
+
sendKeyViaAppleScript(key);
|
|
667
|
+
await sleep(SCROLL_STEP_DELAY);
|
|
668
|
+
}
|
|
669
|
+
await sleep(SCROLL_COMPLETE_DELAY);
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
const ticks = Math.ceil(distance / 100);
|
|
566
673
|
const directionMap = {
|
|
567
674
|
up: [
|
|
568
675
|
0,
|
|
@@ -787,7 +894,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
787
894
|
const tools = new ComputerMidsceneTools();
|
|
788
895
|
runToolsCLI(tools, 'midscene-computer', {
|
|
789
896
|
stripPrefix: 'computer_',
|
|
790
|
-
version: "1.7.
|
|
897
|
+
version: "1.7.5-beta-20260418223706.0",
|
|
791
898
|
extraCommands: createReportCliCommands()
|
|
792
899
|
}).catch((e)=>{
|
|
793
900
|
if (!(e instanceof CLIError)) console.error(e);
|
package/dist/es/index.mjs
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import node_assert from "node:assert";
|
|
2
|
-
import { execFileSync, execSync, spawn } from "node:child_process";
|
|
2
|
+
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
3
4
|
import { createRequire } from "node:module";
|
|
5
|
+
import { dirname, resolve as external_node_path_resolve } from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
4
7
|
import { getMidsceneLocationSchema, z } from "@midscene/core";
|
|
5
8
|
import { actionHoverParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
|
|
6
9
|
import { sleep } from "@midscene/core/utils";
|
|
7
10
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
8
11
|
import { getDebug } from "@midscene/shared/logger";
|
|
9
12
|
import screenshot_desktop from "screenshot-desktop";
|
|
10
|
-
import { existsSync } from "node:fs";
|
|
11
13
|
import { Agent } from "@midscene/core/agent";
|
|
12
14
|
import { BaseMidsceneTools } from "@midscene/shared/mcp";
|
|
13
15
|
import { overrideAIConfig } from "@midscene/shared/env";
|
|
@@ -108,6 +110,45 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
108
110
|
const SCROLL_REPEAT_COUNT = 10;
|
|
109
111
|
const SCROLL_STEP_DELAY = 100;
|
|
110
112
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
113
|
+
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
114
|
+
const EDGE_SCROLL_STEPS = 400;
|
|
115
|
+
const PHASED_PIXELS_PER_STEP = 30;
|
|
116
|
+
const PHASED_MIN_STEPS = 10;
|
|
117
|
+
const APPROX_VIEWPORT_HEIGHT_PX = 600;
|
|
118
|
+
const EDGE_SCROLL_SPEC = {
|
|
119
|
+
scrollToTop: {
|
|
120
|
+
direction: 'up',
|
|
121
|
+
key: 'home',
|
|
122
|
+
libnut: [
|
|
123
|
+
0,
|
|
124
|
+
10
|
|
125
|
+
]
|
|
126
|
+
},
|
|
127
|
+
scrollToBottom: {
|
|
128
|
+
direction: 'down',
|
|
129
|
+
key: 'end',
|
|
130
|
+
libnut: [
|
|
131
|
+
0,
|
|
132
|
+
-10
|
|
133
|
+
]
|
|
134
|
+
},
|
|
135
|
+
scrollToLeft: {
|
|
136
|
+
direction: 'left',
|
|
137
|
+
key: 'home',
|
|
138
|
+
libnut: [
|
|
139
|
+
-10,
|
|
140
|
+
0
|
|
141
|
+
]
|
|
142
|
+
},
|
|
143
|
+
scrollToRight: {
|
|
144
|
+
direction: 'right',
|
|
145
|
+
key: 'end',
|
|
146
|
+
libnut: [
|
|
147
|
+
10,
|
|
148
|
+
0
|
|
149
|
+
]
|
|
150
|
+
}
|
|
151
|
+
};
|
|
111
152
|
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
112
153
|
return: 36,
|
|
113
154
|
enter: 36,
|
|
@@ -186,6 +227,69 @@ async function getLibnut() {
|
|
|
186
227
|
}
|
|
187
228
|
}
|
|
188
229
|
const debugDevice = getDebug('computer:device');
|
|
230
|
+
let phasedScrollBinaryPath;
|
|
231
|
+
function getPhasedScrollBinary() {
|
|
232
|
+
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
233
|
+
if ('darwin' !== process.platform) {
|
|
234
|
+
phasedScrollBinaryPath = null;
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
const require = createRequire(import.meta.url);
|
|
238
|
+
let pkgRoot = null;
|
|
239
|
+
try {
|
|
240
|
+
pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
|
|
241
|
+
} catch {
|
|
242
|
+
const hereDir = dirname(fileURLToPath(import.meta.url));
|
|
243
|
+
for (const candidate of [
|
|
244
|
+
external_node_path_resolve(hereDir, '..'),
|
|
245
|
+
external_node_path_resolve(hereDir, '../..')
|
|
246
|
+
])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
|
|
247
|
+
pkgRoot = candidate;
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (!pkgRoot) {
|
|
252
|
+
debugDevice('phased-scroll: cannot locate @midscene/computer package root');
|
|
253
|
+
phasedScrollBinaryPath = null;
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
|
|
257
|
+
if (!existsSync(binPath)) {
|
|
258
|
+
debugDevice('phased-scroll binary not found at', binPath);
|
|
259
|
+
phasedScrollBinaryPath = null;
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
phasedScrollBinaryPath = binPath;
|
|
263
|
+
return binPath;
|
|
264
|
+
}
|
|
265
|
+
let phasedScrollExecWarned = false;
|
|
266
|
+
function runPhasedScroll(direction, pixels, steps) {
|
|
267
|
+
const bin = getPhasedScrollBinary();
|
|
268
|
+
if (!bin) return false;
|
|
269
|
+
try {
|
|
270
|
+
const res = spawnSync(bin, [
|
|
271
|
+
direction,
|
|
272
|
+
String(Math.max(1, Math.round(pixels))),
|
|
273
|
+
String(steps)
|
|
274
|
+
], {
|
|
275
|
+
stdio: 'ignore'
|
|
276
|
+
});
|
|
277
|
+
if (0 === res.status) return true;
|
|
278
|
+
if (!phasedScrollExecWarned) {
|
|
279
|
+
phasedScrollExecWarned = true;
|
|
280
|
+
console.warn(`[@midscene/computer] phased-scroll helper exited with status ${res.status}; falling back to keyboard/libnut. This usually means Accessibility permission has not been granted to the host process.`);
|
|
281
|
+
}
|
|
282
|
+
debugDevice('phased-scroll exited non-zero', res.status, res.error);
|
|
283
|
+
return false;
|
|
284
|
+
} catch (err) {
|
|
285
|
+
if (!phasedScrollExecWarned) {
|
|
286
|
+
phasedScrollExecWarned = true;
|
|
287
|
+
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
288
|
+
}
|
|
289
|
+
debugDevice('phased-scroll spawn failed', err);
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
189
293
|
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
190
294
|
node_assert(device_libnut, 'libnut not initialized');
|
|
191
295
|
const currentPos = device_libnut.getMousePos();
|
|
@@ -305,7 +409,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
305
409
|
}
|
|
306
410
|
async healthCheck() {
|
|
307
411
|
console.log('[HealthCheck] Starting health check...');
|
|
308
|
-
console.log("[HealthCheck] @midscene/computer v1.7.
|
|
412
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260418223706.0");
|
|
309
413
|
console.log('[HealthCheck] Taking screenshot...');
|
|
310
414
|
const screenshotTimeout = 15000;
|
|
311
415
|
let timeoutId;
|
|
@@ -532,27 +636,15 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
532
636
|
device_libnut.moveMouse(Math.round(x), Math.round(y));
|
|
533
637
|
}
|
|
534
638
|
const scrollType = param?.scrollType;
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
]
|
|
544
|
-
scrollToLeft: [
|
|
545
|
-
-10,
|
|
546
|
-
0
|
|
547
|
-
],
|
|
548
|
-
scrollToRight: [
|
|
549
|
-
10,
|
|
550
|
-
0
|
|
551
|
-
]
|
|
552
|
-
};
|
|
553
|
-
const edgeAction = scrollToEdgeActions[scrollType || ''];
|
|
554
|
-
if (edgeAction) {
|
|
555
|
-
const [dx, dy] = edgeAction;
|
|
639
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
640
|
+
if (edgeSpec) {
|
|
641
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
|
|
642
|
+
if (this.useAppleScript) {
|
|
643
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
644
|
+
await sleep(SCROLL_COMPLETE_DELAY);
|
|
645
|
+
return;
|
|
646
|
+
}
|
|
647
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
556
648
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
557
649
|
device_libnut.scrollMouse(dx, dy);
|
|
558
650
|
await sleep(SCROLL_STEP_DELAY);
|
|
@@ -561,8 +653,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
561
653
|
}
|
|
562
654
|
if ('singleAction' === scrollType || !scrollType) {
|
|
563
655
|
const distance = param?.distance || 500;
|
|
564
|
-
const ticks = Math.ceil(distance / 100);
|
|
565
656
|
const direction = param?.direction || 'down';
|
|
657
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
658
|
+
if (isKnownDirection) {
|
|
659
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
660
|
+
if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
|
|
661
|
+
}
|
|
662
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
663
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
664
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
665
|
+
for(let i = 0; i < pages; i++){
|
|
666
|
+
sendKeyViaAppleScript(key);
|
|
667
|
+
await sleep(SCROLL_STEP_DELAY);
|
|
668
|
+
}
|
|
669
|
+
await sleep(SCROLL_COMPLETE_DELAY);
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
const ticks = Math.ceil(distance / 100);
|
|
566
673
|
const directionMap = {
|
|
567
674
|
up: [
|
|
568
675
|
0,
|
|
@@ -785,7 +892,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
785
892
|
}
|
|
786
893
|
}
|
|
787
894
|
function version() {
|
|
788
|
-
const currentVersion = "1.7.
|
|
895
|
+
const currentVersion = "1.7.5-beta-20260418223706.0";
|
|
789
896
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
790
897
|
return currentVersion;
|
|
791
898
|
}
|
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
import { BaseMCPServer, BaseMidsceneTools, createMCPServerLauncher } from "@midscene/shared/mcp";
|
|
2
2
|
import { Agent } from "@midscene/core/agent";
|
|
3
3
|
import node_assert from "node:assert";
|
|
4
|
-
import { execFileSync, execSync, spawn } from "node:child_process";
|
|
4
|
+
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
5
|
+
import { existsSync } from "node:fs";
|
|
5
6
|
import { createRequire } from "node:module";
|
|
7
|
+
import { dirname, resolve as external_node_path_resolve } from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
6
9
|
import { getMidsceneLocationSchema, z } from "@midscene/core";
|
|
7
10
|
import { actionHoverParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
|
|
8
11
|
import { sleep } from "@midscene/core/utils";
|
|
9
12
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
10
13
|
import { getDebug } from "@midscene/shared/logger";
|
|
11
14
|
import screenshot_desktop from "screenshot-desktop";
|
|
12
|
-
import { existsSync } from "node:fs";
|
|
13
15
|
const debugXvfb = getDebug('computer:xvfb');
|
|
14
16
|
function checkXvfbInstalled() {
|
|
15
17
|
try {
|
|
@@ -107,6 +109,45 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
107
109
|
const SCROLL_REPEAT_COUNT = 10;
|
|
108
110
|
const SCROLL_STEP_DELAY = 100;
|
|
109
111
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
112
|
+
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
113
|
+
const EDGE_SCROLL_STEPS = 400;
|
|
114
|
+
const PHASED_PIXELS_PER_STEP = 30;
|
|
115
|
+
const PHASED_MIN_STEPS = 10;
|
|
116
|
+
const APPROX_VIEWPORT_HEIGHT_PX = 600;
|
|
117
|
+
const EDGE_SCROLL_SPEC = {
|
|
118
|
+
scrollToTop: {
|
|
119
|
+
direction: 'up',
|
|
120
|
+
key: 'home',
|
|
121
|
+
libnut: [
|
|
122
|
+
0,
|
|
123
|
+
10
|
|
124
|
+
]
|
|
125
|
+
},
|
|
126
|
+
scrollToBottom: {
|
|
127
|
+
direction: 'down',
|
|
128
|
+
key: 'end',
|
|
129
|
+
libnut: [
|
|
130
|
+
0,
|
|
131
|
+
-10
|
|
132
|
+
]
|
|
133
|
+
},
|
|
134
|
+
scrollToLeft: {
|
|
135
|
+
direction: 'left',
|
|
136
|
+
key: 'home',
|
|
137
|
+
libnut: [
|
|
138
|
+
-10,
|
|
139
|
+
0
|
|
140
|
+
]
|
|
141
|
+
},
|
|
142
|
+
scrollToRight: {
|
|
143
|
+
direction: 'right',
|
|
144
|
+
key: 'end',
|
|
145
|
+
libnut: [
|
|
146
|
+
10,
|
|
147
|
+
0
|
|
148
|
+
]
|
|
149
|
+
}
|
|
150
|
+
};
|
|
110
151
|
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
111
152
|
return: 36,
|
|
112
153
|
enter: 36,
|
|
@@ -185,6 +226,69 @@ async function getLibnut() {
|
|
|
185
226
|
}
|
|
186
227
|
}
|
|
187
228
|
const debugDevice = getDebug('computer:device');
|
|
229
|
+
let phasedScrollBinaryPath;
|
|
230
|
+
function getPhasedScrollBinary() {
|
|
231
|
+
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
232
|
+
if ('darwin' !== process.platform) {
|
|
233
|
+
phasedScrollBinaryPath = null;
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
const require = createRequire(import.meta.url);
|
|
237
|
+
let pkgRoot = null;
|
|
238
|
+
try {
|
|
239
|
+
pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
|
|
240
|
+
} catch {
|
|
241
|
+
const hereDir = dirname(fileURLToPath(import.meta.url));
|
|
242
|
+
for (const candidate of [
|
|
243
|
+
external_node_path_resolve(hereDir, '..'),
|
|
244
|
+
external_node_path_resolve(hereDir, '../..')
|
|
245
|
+
])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
|
|
246
|
+
pkgRoot = candidate;
|
|
247
|
+
break;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
if (!pkgRoot) {
|
|
251
|
+
debugDevice('phased-scroll: cannot locate @midscene/computer package root');
|
|
252
|
+
phasedScrollBinaryPath = null;
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
|
|
256
|
+
if (!existsSync(binPath)) {
|
|
257
|
+
debugDevice('phased-scroll binary not found at', binPath);
|
|
258
|
+
phasedScrollBinaryPath = null;
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
phasedScrollBinaryPath = binPath;
|
|
262
|
+
return binPath;
|
|
263
|
+
}
|
|
264
|
+
let phasedScrollExecWarned = false;
|
|
265
|
+
function runPhasedScroll(direction, pixels, steps) {
|
|
266
|
+
const bin = getPhasedScrollBinary();
|
|
267
|
+
if (!bin) return false;
|
|
268
|
+
try {
|
|
269
|
+
const res = spawnSync(bin, [
|
|
270
|
+
direction,
|
|
271
|
+
String(Math.max(1, Math.round(pixels))),
|
|
272
|
+
String(steps)
|
|
273
|
+
], {
|
|
274
|
+
stdio: 'ignore'
|
|
275
|
+
});
|
|
276
|
+
if (0 === res.status) return true;
|
|
277
|
+
if (!phasedScrollExecWarned) {
|
|
278
|
+
phasedScrollExecWarned = true;
|
|
279
|
+
console.warn(`[@midscene/computer] phased-scroll helper exited with status ${res.status}; falling back to keyboard/libnut. This usually means Accessibility permission has not been granted to the host process.`);
|
|
280
|
+
}
|
|
281
|
+
debugDevice('phased-scroll exited non-zero', res.status, res.error);
|
|
282
|
+
return false;
|
|
283
|
+
} catch (err) {
|
|
284
|
+
if (!phasedScrollExecWarned) {
|
|
285
|
+
phasedScrollExecWarned = true;
|
|
286
|
+
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
287
|
+
}
|
|
288
|
+
debugDevice('phased-scroll spawn failed', err);
|
|
289
|
+
return false;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
188
292
|
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
189
293
|
node_assert(libnut, 'libnut not initialized');
|
|
190
294
|
const currentPos = libnut.getMousePos();
|
|
@@ -304,7 +408,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
304
408
|
}
|
|
305
409
|
async healthCheck() {
|
|
306
410
|
console.log('[HealthCheck] Starting health check...');
|
|
307
|
-
console.log("[HealthCheck] @midscene/computer v1.7.
|
|
411
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260418223706.0");
|
|
308
412
|
console.log('[HealthCheck] Taking screenshot...');
|
|
309
413
|
const screenshotTimeout = 15000;
|
|
310
414
|
let timeoutId;
|
|
@@ -531,27 +635,15 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
531
635
|
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
532
636
|
}
|
|
533
637
|
const scrollType = param?.scrollType;
|
|
534
|
-
const
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
]
|
|
543
|
-
scrollToLeft: [
|
|
544
|
-
-10,
|
|
545
|
-
0
|
|
546
|
-
],
|
|
547
|
-
scrollToRight: [
|
|
548
|
-
10,
|
|
549
|
-
0
|
|
550
|
-
]
|
|
551
|
-
};
|
|
552
|
-
const edgeAction = scrollToEdgeActions[scrollType || ''];
|
|
553
|
-
if (edgeAction) {
|
|
554
|
-
const [dx, dy] = edgeAction;
|
|
638
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
639
|
+
if (edgeSpec) {
|
|
640
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
|
|
641
|
+
if (this.useAppleScript) {
|
|
642
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
643
|
+
await sleep(SCROLL_COMPLETE_DELAY);
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
555
647
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
556
648
|
libnut.scrollMouse(dx, dy);
|
|
557
649
|
await sleep(SCROLL_STEP_DELAY);
|
|
@@ -560,8 +652,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
560
652
|
}
|
|
561
653
|
if ('singleAction' === scrollType || !scrollType) {
|
|
562
654
|
const distance = param?.distance || 500;
|
|
563
|
-
const ticks = Math.ceil(distance / 100);
|
|
564
655
|
const direction = param?.direction || 'down';
|
|
656
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
657
|
+
if (isKnownDirection) {
|
|
658
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
659
|
+
if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
|
|
660
|
+
}
|
|
661
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
662
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
663
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
664
|
+
for(let i = 0; i < pages; i++){
|
|
665
|
+
sendKeyViaAppleScript(key);
|
|
666
|
+
await sleep(SCROLL_STEP_DELAY);
|
|
667
|
+
}
|
|
668
|
+
await sleep(SCROLL_COMPLETE_DELAY);
|
|
669
|
+
return;
|
|
670
|
+
}
|
|
671
|
+
const ticks = Math.ceil(distance / 100);
|
|
565
672
|
const directionMap = {
|
|
566
673
|
up: [
|
|
567
674
|
0,
|
|
@@ -790,7 +897,7 @@ class ComputerMCPServer extends BaseMCPServer {
|
|
|
790
897
|
constructor(toolsManager){
|
|
791
898
|
super({
|
|
792
899
|
name: '@midscene/computer-mcp',
|
|
793
|
-
version: "1.7.
|
|
900
|
+
version: "1.7.5-beta-20260418223706.0",
|
|
794
901
|
description: 'Control the computer desktop using natural language commands'
|
|
795
902
|
}, toolsManager);
|
|
796
903
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -32,13 +32,15 @@ const agent_namespaceObject = require("@midscene/core/agent");
|
|
|
32
32
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
33
33
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
34
34
|
const external_node_child_process_namespaceObject = require("node:child_process");
|
|
35
|
+
const external_node_fs_namespaceObject = require("node:fs");
|
|
35
36
|
const external_node_module_namespaceObject = require("node:module");
|
|
37
|
+
const external_node_path_namespaceObject = require("node:path");
|
|
38
|
+
const external_node_url_namespaceObject = require("node:url");
|
|
36
39
|
const device_namespaceObject = require("@midscene/core/device");
|
|
37
40
|
const utils_namespaceObject = require("@midscene/core/utils");
|
|
38
41
|
const img_namespaceObject = require("@midscene/shared/img");
|
|
39
42
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
40
43
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
41
|
-
const external_node_fs_namespaceObject = require("node:fs");
|
|
42
44
|
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
43
45
|
function checkXvfbInstalled() {
|
|
44
46
|
try {
|
|
@@ -136,6 +138,45 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
136
138
|
const SCROLL_REPEAT_COUNT = 10;
|
|
137
139
|
const SCROLL_STEP_DELAY = 100;
|
|
138
140
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
141
|
+
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
142
|
+
const EDGE_SCROLL_STEPS = 400;
|
|
143
|
+
const PHASED_PIXELS_PER_STEP = 30;
|
|
144
|
+
const PHASED_MIN_STEPS = 10;
|
|
145
|
+
const APPROX_VIEWPORT_HEIGHT_PX = 600;
|
|
146
|
+
const EDGE_SCROLL_SPEC = {
|
|
147
|
+
scrollToTop: {
|
|
148
|
+
direction: 'up',
|
|
149
|
+
key: 'home',
|
|
150
|
+
libnut: [
|
|
151
|
+
0,
|
|
152
|
+
10
|
|
153
|
+
]
|
|
154
|
+
},
|
|
155
|
+
scrollToBottom: {
|
|
156
|
+
direction: 'down',
|
|
157
|
+
key: 'end',
|
|
158
|
+
libnut: [
|
|
159
|
+
0,
|
|
160
|
+
-10
|
|
161
|
+
]
|
|
162
|
+
},
|
|
163
|
+
scrollToLeft: {
|
|
164
|
+
direction: 'left',
|
|
165
|
+
key: 'home',
|
|
166
|
+
libnut: [
|
|
167
|
+
-10,
|
|
168
|
+
0
|
|
169
|
+
]
|
|
170
|
+
},
|
|
171
|
+
scrollToRight: {
|
|
172
|
+
direction: 'right',
|
|
173
|
+
key: 'end',
|
|
174
|
+
libnut: [
|
|
175
|
+
10,
|
|
176
|
+
0
|
|
177
|
+
]
|
|
178
|
+
}
|
|
179
|
+
};
|
|
139
180
|
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
140
181
|
return: 36,
|
|
141
182
|
enter: 36,
|
|
@@ -214,6 +255,69 @@ async function getLibnut() {
|
|
|
214
255
|
}
|
|
215
256
|
}
|
|
216
257
|
const debugDevice = (0, logger_namespaceObject.getDebug)('computer:device');
|
|
258
|
+
let phasedScrollBinaryPath;
|
|
259
|
+
function getPhasedScrollBinary() {
|
|
260
|
+
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
261
|
+
if ('darwin' !== process.platform) {
|
|
262
|
+
phasedScrollBinaryPath = null;
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
|
|
266
|
+
let pkgRoot = null;
|
|
267
|
+
try {
|
|
268
|
+
pkgRoot = (0, external_node_path_namespaceObject.dirname)(require1.resolve('@midscene/computer/package.json'));
|
|
269
|
+
} catch {
|
|
270
|
+
const hereDir = (0, external_node_path_namespaceObject.dirname)((0, external_node_url_namespaceObject.fileURLToPath)(__rslib_import_meta_url__));
|
|
271
|
+
for (const candidate of [
|
|
272
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '..'),
|
|
273
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '../..')
|
|
274
|
+
])if ((0, external_node_fs_namespaceObject.existsSync)((0, external_node_path_namespaceObject.resolve)(candidate, 'package.json'))) {
|
|
275
|
+
pkgRoot = candidate;
|
|
276
|
+
break;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
if (!pkgRoot) {
|
|
280
|
+
debugDevice('phased-scroll: cannot locate @midscene/computer package root');
|
|
281
|
+
phasedScrollBinaryPath = null;
|
|
282
|
+
return null;
|
|
283
|
+
}
|
|
284
|
+
const binPath = (0, external_node_path_namespaceObject.resolve)(pkgRoot, 'bin/darwin/phased-scroll');
|
|
285
|
+
if (!(0, external_node_fs_namespaceObject.existsSync)(binPath)) {
|
|
286
|
+
debugDevice('phased-scroll binary not found at', binPath);
|
|
287
|
+
phasedScrollBinaryPath = null;
|
|
288
|
+
return null;
|
|
289
|
+
}
|
|
290
|
+
phasedScrollBinaryPath = binPath;
|
|
291
|
+
return binPath;
|
|
292
|
+
}
|
|
293
|
+
let phasedScrollExecWarned = false;
|
|
294
|
+
function runPhasedScroll(direction, pixels, steps) {
|
|
295
|
+
const bin = getPhasedScrollBinary();
|
|
296
|
+
if (!bin) return false;
|
|
297
|
+
try {
|
|
298
|
+
const res = (0, external_node_child_process_namespaceObject.spawnSync)(bin, [
|
|
299
|
+
direction,
|
|
300
|
+
String(Math.max(1, Math.round(pixels))),
|
|
301
|
+
String(steps)
|
|
302
|
+
], {
|
|
303
|
+
stdio: 'ignore'
|
|
304
|
+
});
|
|
305
|
+
if (0 === res.status) return true;
|
|
306
|
+
if (!phasedScrollExecWarned) {
|
|
307
|
+
phasedScrollExecWarned = true;
|
|
308
|
+
console.warn(`[@midscene/computer] phased-scroll helper exited with status ${res.status}; falling back to keyboard/libnut. This usually means Accessibility permission has not been granted to the host process.`);
|
|
309
|
+
}
|
|
310
|
+
debugDevice('phased-scroll exited non-zero', res.status, res.error);
|
|
311
|
+
return false;
|
|
312
|
+
} catch (err) {
|
|
313
|
+
if (!phasedScrollExecWarned) {
|
|
314
|
+
phasedScrollExecWarned = true;
|
|
315
|
+
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
316
|
+
}
|
|
317
|
+
debugDevice('phased-scroll spawn failed', err);
|
|
318
|
+
return false;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
217
321
|
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
218
322
|
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
219
323
|
const currentPos = libnut.getMousePos();
|
|
@@ -333,7 +437,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
333
437
|
}
|
|
334
438
|
async healthCheck() {
|
|
335
439
|
console.log('[HealthCheck] Starting health check...');
|
|
336
|
-
console.log("[HealthCheck] @midscene/computer v1.7.
|
|
440
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260418223706.0");
|
|
337
441
|
console.log('[HealthCheck] Taking screenshot...');
|
|
338
442
|
const screenshotTimeout = 15000;
|
|
339
443
|
let timeoutId;
|
|
@@ -560,27 +664,15 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
560
664
|
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
561
665
|
}
|
|
562
666
|
const scrollType = param?.scrollType;
|
|
563
|
-
const
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
]
|
|
572
|
-
scrollToLeft: [
|
|
573
|
-
-10,
|
|
574
|
-
0
|
|
575
|
-
],
|
|
576
|
-
scrollToRight: [
|
|
577
|
-
10,
|
|
578
|
-
0
|
|
579
|
-
]
|
|
580
|
-
};
|
|
581
|
-
const edgeAction = scrollToEdgeActions[scrollType || ''];
|
|
582
|
-
if (edgeAction) {
|
|
583
|
-
const [dx, dy] = edgeAction;
|
|
667
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
668
|
+
if (edgeSpec) {
|
|
669
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
670
|
+
if (this.useAppleScript) {
|
|
671
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
672
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
584
676
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
585
677
|
libnut.scrollMouse(dx, dy);
|
|
586
678
|
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
@@ -589,8 +681,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
589
681
|
}
|
|
590
682
|
if ('singleAction' === scrollType || !scrollType) {
|
|
591
683
|
const distance = param?.distance || 500;
|
|
592
|
-
const ticks = Math.ceil(distance / 100);
|
|
593
684
|
const direction = param?.direction || 'down';
|
|
685
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
686
|
+
if (isKnownDirection) {
|
|
687
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
688
|
+
if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
689
|
+
}
|
|
690
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
691
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
692
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
693
|
+
for(let i = 0; i < pages; i++){
|
|
694
|
+
sendKeyViaAppleScript(key);
|
|
695
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
696
|
+
}
|
|
697
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
698
|
+
return;
|
|
699
|
+
}
|
|
700
|
+
const ticks = Math.ceil(distance / 100);
|
|
594
701
|
const directionMap = {
|
|
595
702
|
up: [
|
|
596
703
|
0,
|
|
@@ -815,7 +922,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
815
922
|
const tools = new ComputerMidsceneTools();
|
|
816
923
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
817
924
|
stripPrefix: 'computer_',
|
|
818
|
-
version: "1.7.
|
|
925
|
+
version: "1.7.5-beta-20260418223706.0",
|
|
819
926
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
820
927
|
}).catch((e)=>{
|
|
821
928
|
if (!(e instanceof cli_namespaceObject.CLIError)) console.error(e);
|
package/dist/lib/index.js
CHANGED
|
@@ -51,7 +51,10 @@ __webpack_require__.d(__webpack_exports__, {
|
|
|
51
51
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
52
52
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
53
53
|
const external_node_child_process_namespaceObject = require("node:child_process");
|
|
54
|
+
const external_node_fs_namespaceObject = require("node:fs");
|
|
54
55
|
const external_node_module_namespaceObject = require("node:module");
|
|
56
|
+
const external_node_path_namespaceObject = require("node:path");
|
|
57
|
+
const external_node_url_namespaceObject = require("node:url");
|
|
55
58
|
const core_namespaceObject = require("@midscene/core");
|
|
56
59
|
const device_namespaceObject = require("@midscene/core/device");
|
|
57
60
|
const utils_namespaceObject = require("@midscene/core/utils");
|
|
@@ -59,7 +62,6 @@ const img_namespaceObject = require("@midscene/shared/img");
|
|
|
59
62
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
60
63
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
61
64
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
62
|
-
const external_node_fs_namespaceObject = require("node:fs");
|
|
63
65
|
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
64
66
|
function checkXvfbInstalled() {
|
|
65
67
|
try {
|
|
@@ -157,6 +159,45 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
157
159
|
const SCROLL_REPEAT_COUNT = 10;
|
|
158
160
|
const SCROLL_STEP_DELAY = 100;
|
|
159
161
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
162
|
+
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
163
|
+
const EDGE_SCROLL_STEPS = 400;
|
|
164
|
+
const PHASED_PIXELS_PER_STEP = 30;
|
|
165
|
+
const PHASED_MIN_STEPS = 10;
|
|
166
|
+
const APPROX_VIEWPORT_HEIGHT_PX = 600;
|
|
167
|
+
const EDGE_SCROLL_SPEC = {
|
|
168
|
+
scrollToTop: {
|
|
169
|
+
direction: 'up',
|
|
170
|
+
key: 'home',
|
|
171
|
+
libnut: [
|
|
172
|
+
0,
|
|
173
|
+
10
|
|
174
|
+
]
|
|
175
|
+
},
|
|
176
|
+
scrollToBottom: {
|
|
177
|
+
direction: 'down',
|
|
178
|
+
key: 'end',
|
|
179
|
+
libnut: [
|
|
180
|
+
0,
|
|
181
|
+
-10
|
|
182
|
+
]
|
|
183
|
+
},
|
|
184
|
+
scrollToLeft: {
|
|
185
|
+
direction: 'left',
|
|
186
|
+
key: 'home',
|
|
187
|
+
libnut: [
|
|
188
|
+
-10,
|
|
189
|
+
0
|
|
190
|
+
]
|
|
191
|
+
},
|
|
192
|
+
scrollToRight: {
|
|
193
|
+
direction: 'right',
|
|
194
|
+
key: 'end',
|
|
195
|
+
libnut: [
|
|
196
|
+
10,
|
|
197
|
+
0
|
|
198
|
+
]
|
|
199
|
+
}
|
|
200
|
+
};
|
|
160
201
|
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
161
202
|
return: 36,
|
|
162
203
|
enter: 36,
|
|
@@ -235,6 +276,69 @@ async function getLibnut() {
|
|
|
235
276
|
}
|
|
236
277
|
}
|
|
237
278
|
const debugDevice = (0, logger_namespaceObject.getDebug)('computer:device');
|
|
279
|
+
let phasedScrollBinaryPath;
|
|
280
|
+
function getPhasedScrollBinary() {
|
|
281
|
+
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
282
|
+
if ('darwin' !== process.platform) {
|
|
283
|
+
phasedScrollBinaryPath = null;
|
|
284
|
+
return null;
|
|
285
|
+
}
|
|
286
|
+
const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
|
|
287
|
+
let pkgRoot = null;
|
|
288
|
+
try {
|
|
289
|
+
pkgRoot = (0, external_node_path_namespaceObject.dirname)(require1.resolve('@midscene/computer/package.json'));
|
|
290
|
+
} catch {
|
|
291
|
+
const hereDir = (0, external_node_path_namespaceObject.dirname)((0, external_node_url_namespaceObject.fileURLToPath)(__rslib_import_meta_url__));
|
|
292
|
+
for (const candidate of [
|
|
293
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '..'),
|
|
294
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '../..')
|
|
295
|
+
])if ((0, external_node_fs_namespaceObject.existsSync)((0, external_node_path_namespaceObject.resolve)(candidate, 'package.json'))) {
|
|
296
|
+
pkgRoot = candidate;
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
if (!pkgRoot) {
|
|
301
|
+
debugDevice('phased-scroll: cannot locate @midscene/computer package root');
|
|
302
|
+
phasedScrollBinaryPath = null;
|
|
303
|
+
return null;
|
|
304
|
+
}
|
|
305
|
+
const binPath = (0, external_node_path_namespaceObject.resolve)(pkgRoot, 'bin/darwin/phased-scroll');
|
|
306
|
+
if (!(0, external_node_fs_namespaceObject.existsSync)(binPath)) {
|
|
307
|
+
debugDevice('phased-scroll binary not found at', binPath);
|
|
308
|
+
phasedScrollBinaryPath = null;
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
phasedScrollBinaryPath = binPath;
|
|
312
|
+
return binPath;
|
|
313
|
+
}
|
|
314
|
+
let phasedScrollExecWarned = false;
|
|
315
|
+
function runPhasedScroll(direction, pixels, steps) {
|
|
316
|
+
const bin = getPhasedScrollBinary();
|
|
317
|
+
if (!bin) return false;
|
|
318
|
+
try {
|
|
319
|
+
const res = (0, external_node_child_process_namespaceObject.spawnSync)(bin, [
|
|
320
|
+
direction,
|
|
321
|
+
String(Math.max(1, Math.round(pixels))),
|
|
322
|
+
String(steps)
|
|
323
|
+
], {
|
|
324
|
+
stdio: 'ignore'
|
|
325
|
+
});
|
|
326
|
+
if (0 === res.status) return true;
|
|
327
|
+
if (!phasedScrollExecWarned) {
|
|
328
|
+
phasedScrollExecWarned = true;
|
|
329
|
+
console.warn(`[@midscene/computer] phased-scroll helper exited with status ${res.status}; falling back to keyboard/libnut. This usually means Accessibility permission has not been granted to the host process.`);
|
|
330
|
+
}
|
|
331
|
+
debugDevice('phased-scroll exited non-zero', res.status, res.error);
|
|
332
|
+
return false;
|
|
333
|
+
} catch (err) {
|
|
334
|
+
if (!phasedScrollExecWarned) {
|
|
335
|
+
phasedScrollExecWarned = true;
|
|
336
|
+
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
337
|
+
}
|
|
338
|
+
debugDevice('phased-scroll spawn failed', err);
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
238
342
|
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
239
343
|
external_node_assert_default()(device_libnut, 'libnut not initialized');
|
|
240
344
|
const currentPos = device_libnut.getMousePos();
|
|
@@ -354,7 +458,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
354
458
|
}
|
|
355
459
|
async healthCheck() {
|
|
356
460
|
console.log('[HealthCheck] Starting health check...');
|
|
357
|
-
console.log("[HealthCheck] @midscene/computer v1.7.
|
|
461
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260418223706.0");
|
|
358
462
|
console.log('[HealthCheck] Taking screenshot...');
|
|
359
463
|
const screenshotTimeout = 15000;
|
|
360
464
|
let timeoutId;
|
|
@@ -581,27 +685,15 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
581
685
|
device_libnut.moveMouse(Math.round(x), Math.round(y));
|
|
582
686
|
}
|
|
583
687
|
const scrollType = param?.scrollType;
|
|
584
|
-
const
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
]
|
|
593
|
-
scrollToLeft: [
|
|
594
|
-
-10,
|
|
595
|
-
0
|
|
596
|
-
],
|
|
597
|
-
scrollToRight: [
|
|
598
|
-
10,
|
|
599
|
-
0
|
|
600
|
-
]
|
|
601
|
-
};
|
|
602
|
-
const edgeAction = scrollToEdgeActions[scrollType || ''];
|
|
603
|
-
if (edgeAction) {
|
|
604
|
-
const [dx, dy] = edgeAction;
|
|
688
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
689
|
+
if (edgeSpec) {
|
|
690
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
691
|
+
if (this.useAppleScript) {
|
|
692
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
693
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
605
697
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
606
698
|
device_libnut.scrollMouse(dx, dy);
|
|
607
699
|
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
@@ -610,8 +702,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
610
702
|
}
|
|
611
703
|
if ('singleAction' === scrollType || !scrollType) {
|
|
612
704
|
const distance = param?.distance || 500;
|
|
613
|
-
const ticks = Math.ceil(distance / 100);
|
|
614
705
|
const direction = param?.direction || 'down';
|
|
706
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
707
|
+
if (isKnownDirection) {
|
|
708
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
709
|
+
if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
710
|
+
}
|
|
711
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
712
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
713
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
714
|
+
for(let i = 0; i < pages; i++){
|
|
715
|
+
sendKeyViaAppleScript(key);
|
|
716
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
717
|
+
}
|
|
718
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
const ticks = Math.ceil(distance / 100);
|
|
615
722
|
const directionMap = {
|
|
616
723
|
up: [
|
|
617
724
|
0,
|
|
@@ -837,7 +944,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
837
944
|
}
|
|
838
945
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
839
946
|
function version() {
|
|
840
|
-
const currentVersion = "1.7.
|
|
947
|
+
const currentVersion = "1.7.5-beta-20260418223706.0";
|
|
841
948
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
842
949
|
return currentVersion;
|
|
843
950
|
}
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -45,7 +45,10 @@ const agent_namespaceObject = require("@midscene/core/agent");
|
|
|
45
45
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
46
46
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
47
47
|
const external_node_child_process_namespaceObject = require("node:child_process");
|
|
48
|
+
const external_node_fs_namespaceObject = require("node:fs");
|
|
48
49
|
const external_node_module_namespaceObject = require("node:module");
|
|
50
|
+
const external_node_path_namespaceObject = require("node:path");
|
|
51
|
+
const external_node_url_namespaceObject = require("node:url");
|
|
49
52
|
const core_namespaceObject = require("@midscene/core");
|
|
50
53
|
const device_namespaceObject = require("@midscene/core/device");
|
|
51
54
|
const utils_namespaceObject = require("@midscene/core/utils");
|
|
@@ -53,7 +56,6 @@ const img_namespaceObject = require("@midscene/shared/img");
|
|
|
53
56
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
54
57
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
55
58
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
56
|
-
const external_node_fs_namespaceObject = require("node:fs");
|
|
57
59
|
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
58
60
|
function checkXvfbInstalled() {
|
|
59
61
|
try {
|
|
@@ -151,6 +153,45 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
151
153
|
const SCROLL_REPEAT_COUNT = 10;
|
|
152
154
|
const SCROLL_STEP_DELAY = 100;
|
|
153
155
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
156
|
+
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
157
|
+
const EDGE_SCROLL_STEPS = 400;
|
|
158
|
+
const PHASED_PIXELS_PER_STEP = 30;
|
|
159
|
+
const PHASED_MIN_STEPS = 10;
|
|
160
|
+
const APPROX_VIEWPORT_HEIGHT_PX = 600;
|
|
161
|
+
const EDGE_SCROLL_SPEC = {
|
|
162
|
+
scrollToTop: {
|
|
163
|
+
direction: 'up',
|
|
164
|
+
key: 'home',
|
|
165
|
+
libnut: [
|
|
166
|
+
0,
|
|
167
|
+
10
|
|
168
|
+
]
|
|
169
|
+
},
|
|
170
|
+
scrollToBottom: {
|
|
171
|
+
direction: 'down',
|
|
172
|
+
key: 'end',
|
|
173
|
+
libnut: [
|
|
174
|
+
0,
|
|
175
|
+
-10
|
|
176
|
+
]
|
|
177
|
+
},
|
|
178
|
+
scrollToLeft: {
|
|
179
|
+
direction: 'left',
|
|
180
|
+
key: 'home',
|
|
181
|
+
libnut: [
|
|
182
|
+
-10,
|
|
183
|
+
0
|
|
184
|
+
]
|
|
185
|
+
},
|
|
186
|
+
scrollToRight: {
|
|
187
|
+
direction: 'right',
|
|
188
|
+
key: 'end',
|
|
189
|
+
libnut: [
|
|
190
|
+
10,
|
|
191
|
+
0
|
|
192
|
+
]
|
|
193
|
+
}
|
|
194
|
+
};
|
|
154
195
|
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
155
196
|
return: 36,
|
|
156
197
|
enter: 36,
|
|
@@ -229,6 +270,69 @@ async function getLibnut() {
|
|
|
229
270
|
}
|
|
230
271
|
}
|
|
231
272
|
const debugDevice = (0, logger_namespaceObject.getDebug)('computer:device');
|
|
273
|
+
let phasedScrollBinaryPath;
|
|
274
|
+
function getPhasedScrollBinary() {
|
|
275
|
+
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
276
|
+
if ('darwin' !== process.platform) {
|
|
277
|
+
phasedScrollBinaryPath = null;
|
|
278
|
+
return null;
|
|
279
|
+
}
|
|
280
|
+
const require1 = (0, external_node_module_namespaceObject.createRequire)(__rslib_import_meta_url__);
|
|
281
|
+
let pkgRoot = null;
|
|
282
|
+
try {
|
|
283
|
+
pkgRoot = (0, external_node_path_namespaceObject.dirname)(require1.resolve('@midscene/computer/package.json'));
|
|
284
|
+
} catch {
|
|
285
|
+
const hereDir = (0, external_node_path_namespaceObject.dirname)((0, external_node_url_namespaceObject.fileURLToPath)(__rslib_import_meta_url__));
|
|
286
|
+
for (const candidate of [
|
|
287
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '..'),
|
|
288
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '../..')
|
|
289
|
+
])if ((0, external_node_fs_namespaceObject.existsSync)((0, external_node_path_namespaceObject.resolve)(candidate, 'package.json'))) {
|
|
290
|
+
pkgRoot = candidate;
|
|
291
|
+
break;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
if (!pkgRoot) {
|
|
295
|
+
debugDevice('phased-scroll: cannot locate @midscene/computer package root');
|
|
296
|
+
phasedScrollBinaryPath = null;
|
|
297
|
+
return null;
|
|
298
|
+
}
|
|
299
|
+
const binPath = (0, external_node_path_namespaceObject.resolve)(pkgRoot, 'bin/darwin/phased-scroll');
|
|
300
|
+
if (!(0, external_node_fs_namespaceObject.existsSync)(binPath)) {
|
|
301
|
+
debugDevice('phased-scroll binary not found at', binPath);
|
|
302
|
+
phasedScrollBinaryPath = null;
|
|
303
|
+
return null;
|
|
304
|
+
}
|
|
305
|
+
phasedScrollBinaryPath = binPath;
|
|
306
|
+
return binPath;
|
|
307
|
+
}
|
|
308
|
+
let phasedScrollExecWarned = false;
|
|
309
|
+
function runPhasedScroll(direction, pixels, steps) {
|
|
310
|
+
const bin = getPhasedScrollBinary();
|
|
311
|
+
if (!bin) return false;
|
|
312
|
+
try {
|
|
313
|
+
const res = (0, external_node_child_process_namespaceObject.spawnSync)(bin, [
|
|
314
|
+
direction,
|
|
315
|
+
String(Math.max(1, Math.round(pixels))),
|
|
316
|
+
String(steps)
|
|
317
|
+
], {
|
|
318
|
+
stdio: 'ignore'
|
|
319
|
+
});
|
|
320
|
+
if (0 === res.status) return true;
|
|
321
|
+
if (!phasedScrollExecWarned) {
|
|
322
|
+
phasedScrollExecWarned = true;
|
|
323
|
+
console.warn(`[@midscene/computer] phased-scroll helper exited with status ${res.status}; falling back to keyboard/libnut. This usually means Accessibility permission has not been granted to the host process.`);
|
|
324
|
+
}
|
|
325
|
+
debugDevice('phased-scroll exited non-zero', res.status, res.error);
|
|
326
|
+
return false;
|
|
327
|
+
} catch (err) {
|
|
328
|
+
if (!phasedScrollExecWarned) {
|
|
329
|
+
phasedScrollExecWarned = true;
|
|
330
|
+
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
331
|
+
}
|
|
332
|
+
debugDevice('phased-scroll spawn failed', err);
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
232
336
|
async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
233
337
|
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
234
338
|
const currentPos = libnut.getMousePos();
|
|
@@ -348,7 +452,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
348
452
|
}
|
|
349
453
|
async healthCheck() {
|
|
350
454
|
console.log('[HealthCheck] Starting health check...');
|
|
351
|
-
console.log("[HealthCheck] @midscene/computer v1.7.
|
|
455
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260418223706.0");
|
|
352
456
|
console.log('[HealthCheck] Taking screenshot...');
|
|
353
457
|
const screenshotTimeout = 15000;
|
|
354
458
|
let timeoutId;
|
|
@@ -575,27 +679,15 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
575
679
|
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
576
680
|
}
|
|
577
681
|
const scrollType = param?.scrollType;
|
|
578
|
-
const
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
]
|
|
587
|
-
scrollToLeft: [
|
|
588
|
-
-10,
|
|
589
|
-
0
|
|
590
|
-
],
|
|
591
|
-
scrollToRight: [
|
|
592
|
-
10,
|
|
593
|
-
0
|
|
594
|
-
]
|
|
595
|
-
};
|
|
596
|
-
const edgeAction = scrollToEdgeActions[scrollType || ''];
|
|
597
|
-
if (edgeAction) {
|
|
598
|
-
const [dx, dy] = edgeAction;
|
|
682
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
683
|
+
if (edgeSpec) {
|
|
684
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
685
|
+
if (this.useAppleScript) {
|
|
686
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
687
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
599
691
|
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
600
692
|
libnut.scrollMouse(dx, dy);
|
|
601
693
|
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
@@ -604,8 +696,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
604
696
|
}
|
|
605
697
|
if ('singleAction' === scrollType || !scrollType) {
|
|
606
698
|
const distance = param?.distance || 500;
|
|
607
|
-
const ticks = Math.ceil(distance / 100);
|
|
608
699
|
const direction = param?.direction || 'down';
|
|
700
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
701
|
+
if (isKnownDirection) {
|
|
702
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
703
|
+
if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
704
|
+
}
|
|
705
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
706
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
707
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
708
|
+
for(let i = 0; i < pages; i++){
|
|
709
|
+
sendKeyViaAppleScript(key);
|
|
710
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
711
|
+
}
|
|
712
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
const ticks = Math.ceil(distance / 100);
|
|
609
716
|
const directionMap = {
|
|
610
717
|
up: [
|
|
611
718
|
0,
|
|
@@ -834,7 +941,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
834
941
|
constructor(toolsManager){
|
|
835
942
|
super({
|
|
836
943
|
name: '@midscene/computer-mcp',
|
|
837
|
-
version: "1.7.
|
|
944
|
+
version: "1.7.5-beta-20260418223706.0",
|
|
838
945
|
description: 'Control the computer desktop using natural language commands'
|
|
839
946
|
}, toolsManager);
|
|
840
947
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.5-beta-20260418223706.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -36,8 +36,8 @@
|
|
|
36
36
|
"@computer-use/libnut": "^4.2.0",
|
|
37
37
|
"clipboardy": "^4.0.0",
|
|
38
38
|
"screenshot-desktop": "^1.15.3",
|
|
39
|
-
"@midscene/core": "1.7.
|
|
40
|
-
"@midscene/shared": "1.7.
|
|
39
|
+
"@midscene/core": "1.7.5-beta-20260418223706.0",
|
|
40
|
+
"@midscene/shared": "1.7.5-beta-20260418223706.0"
|
|
41
41
|
},
|
|
42
42
|
"optionalDependencies": {
|
|
43
43
|
"node-mac-permissions": "2.5.0"
|
|
@@ -53,6 +53,7 @@
|
|
|
53
53
|
},
|
|
54
54
|
"scripts": {
|
|
55
55
|
"build": "rslib build",
|
|
56
|
+
"build:native": "mkdir -p bin/darwin && clang -O2 -arch arm64 -arch x86_64 -framework ApplicationServices -o bin/darwin/phased-scroll native/phased-scroll.m",
|
|
56
57
|
"dev": "rslib build --watch",
|
|
57
58
|
"test": "vitest run",
|
|
58
59
|
"test:ai": "AI_TEST_TYPE=computer vitest run"
|