@midscene/computer 1.4.6 → 1.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +145 -39
- package/dist/es/index.mjs +132 -32
- package/dist/es/mcp-server.mjs +145 -39
- package/dist/lib/cli.js +144 -38
- package/dist/lib/index.js +138 -32
- package/dist/lib/mcp-server.js +144 -38
- package/dist/types/index.d.ts +25 -0
- package/dist/types/mcp-server.d.ts +14 -1
- package/package.json +3 -3
package/dist/lib/index.js
CHANGED
|
@@ -36,12 +36,14 @@ var __webpack_require__ = {};
|
|
|
36
36
|
var __webpack_exports__ = {};
|
|
37
37
|
__webpack_require__.r(__webpack_exports__);
|
|
38
38
|
__webpack_require__.d(__webpack_exports__, {
|
|
39
|
+
checkComputerEnvironment: ()=>checkComputerEnvironment,
|
|
40
|
+
checkXvfbInstalled: ()=>checkXvfbInstalled,
|
|
39
41
|
checkAccessibilityPermission: ()=>checkAccessibilityPermission,
|
|
42
|
+
agentFromComputer: ()=>agentFromComputer,
|
|
43
|
+
needsXvfb: ()=>needsXvfb,
|
|
40
44
|
overrideAIConfig: ()=>env_namespaceObject.overrideAIConfig,
|
|
41
45
|
getConnectedDisplays: ()=>getConnectedDisplays,
|
|
42
|
-
checkComputerEnvironment: ()=>checkComputerEnvironment,
|
|
43
46
|
ComputerDevice: ()=>ComputerDevice,
|
|
44
|
-
agentFromComputer: ()=>agentFromComputer,
|
|
45
47
|
ComputerAgent: ()=>ComputerAgent
|
|
46
48
|
});
|
|
47
49
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
@@ -55,6 +57,74 @@ const img_namespaceObject = require("@midscene/shared/img");
|
|
|
55
57
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
56
58
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
57
59
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
60
|
+
const external_node_fs_namespaceObject = require("node:fs");
|
|
61
|
+
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
62
|
+
function checkXvfbInstalled() {
|
|
63
|
+
try {
|
|
64
|
+
(0, external_node_child_process_namespaceObject.execSync)('which Xvfb', {
|
|
65
|
+
stdio: 'ignore'
|
|
66
|
+
});
|
|
67
|
+
return true;
|
|
68
|
+
} catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function findAvailableDisplay(startFrom = 99) {
|
|
73
|
+
for(let n = startFrom; n < startFrom + 100; n++)if (!(0, external_node_fs_namespaceObject.existsSync)(`/tmp/.X${n}-lock`)) return n;
|
|
74
|
+
throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
|
|
75
|
+
}
|
|
76
|
+
function startXvfb(options) {
|
|
77
|
+
const resolution = options?.resolution || '1920x1080x24';
|
|
78
|
+
const displayNum = options?.displayNumber ?? findAvailableDisplay();
|
|
79
|
+
const display = `:${displayNum}`;
|
|
80
|
+
return new Promise((resolve, reject)=>{
|
|
81
|
+
debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
|
|
82
|
+
const xvfbProcess = (0, external_node_child_process_namespaceObject.spawn)('Xvfb', [
|
|
83
|
+
display,
|
|
84
|
+
'-screen',
|
|
85
|
+
'0',
|
|
86
|
+
resolution,
|
|
87
|
+
'-ac',
|
|
88
|
+
'-nolisten',
|
|
89
|
+
'tcp'
|
|
90
|
+
], {
|
|
91
|
+
stdio: 'ignore'
|
|
92
|
+
});
|
|
93
|
+
let settled = false;
|
|
94
|
+
xvfbProcess.on('error', (err)=>{
|
|
95
|
+
if (!settled) {
|
|
96
|
+
settled = true;
|
|
97
|
+
reject(new Error(`Failed to start Xvfb: ${err.message}`));
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
xvfbProcess.on('exit', (code)=>{
|
|
101
|
+
if (!settled) {
|
|
102
|
+
settled = true;
|
|
103
|
+
reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
const instance = {
|
|
107
|
+
process: xvfbProcess,
|
|
108
|
+
display,
|
|
109
|
+
stop () {
|
|
110
|
+
try {
|
|
111
|
+
xvfbProcess.kill('SIGTERM');
|
|
112
|
+
} catch {}
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
setTimeout(()=>{
|
|
116
|
+
if (!settled) {
|
|
117
|
+
settled = true;
|
|
118
|
+
debugXvfb(`Xvfb started on display ${display}`);
|
|
119
|
+
resolve(instance);
|
|
120
|
+
}
|
|
121
|
+
}, 500);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
function needsXvfb(explicitOpt) {
|
|
125
|
+
if ('linux' !== process.platform) return false;
|
|
126
|
+
return true === explicitOpt;
|
|
127
|
+
}
|
|
58
128
|
function _define_property(obj, key, value) {
|
|
59
129
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
60
130
|
value: value,
|
|
@@ -225,53 +295,74 @@ class ComputerDevice {
|
|
|
225
295
|
async connect() {
|
|
226
296
|
debugDevice('Connecting to computer device');
|
|
227
297
|
try {
|
|
298
|
+
const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
|
|
299
|
+
if (needsXvfb(headless)) {
|
|
300
|
+
if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
|
|
301
|
+
this.xvfbInstance = await startXvfb({
|
|
302
|
+
resolution: this.options?.xvfbResolution
|
|
303
|
+
});
|
|
304
|
+
process.env.DISPLAY = this.xvfbInstance.display;
|
|
305
|
+
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
306
|
+
this.xvfbCleanup = ()=>{
|
|
307
|
+
if (this.xvfbInstance) {
|
|
308
|
+
this.xvfbInstance.stop();
|
|
309
|
+
this.xvfbInstance = void 0;
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
process.on('exit', this.xvfbCleanup);
|
|
313
|
+
process.on('SIGINT', this.xvfbCleanup);
|
|
314
|
+
process.on('SIGTERM', this.xvfbCleanup);
|
|
315
|
+
}
|
|
228
316
|
device_libnut = await getLibnut();
|
|
229
317
|
const size = await this.size();
|
|
230
318
|
const displays = await ComputerDevice.listDisplays();
|
|
319
|
+
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
231
320
|
this.description = `
|
|
232
321
|
Type: Computer
|
|
233
322
|
Platform: ${process.platform}
|
|
234
323
|
Display: ${this.displayId || 'Primary'}
|
|
235
324
|
Screen Size: ${size.width}x${size.height}
|
|
236
|
-
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}
|
|
325
|
+
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
237
326
|
`;
|
|
238
327
|
debugDevice('Computer device connected', this.description);
|
|
328
|
+
await this.healthCheck();
|
|
239
329
|
} catch (error) {
|
|
330
|
+
if (this.xvfbInstance) {
|
|
331
|
+
this.xvfbInstance.stop();
|
|
332
|
+
this.xvfbInstance = void 0;
|
|
333
|
+
}
|
|
240
334
|
debugDevice(`Failed to connect: ${error}`);
|
|
241
335
|
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
242
336
|
}
|
|
243
|
-
await this.healthCheck();
|
|
244
337
|
}
|
|
245
338
|
async healthCheck() {
|
|
246
339
|
console.log('[HealthCheck] Starting health check...');
|
|
247
340
|
console.log('[HealthCheck] Taking screenshot...');
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
341
|
+
const screenshotTimeout = 15000;
|
|
342
|
+
let timeoutId;
|
|
343
|
+
const timeoutPromise = new Promise((_, reject)=>{
|
|
344
|
+
timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
|
|
345
|
+
});
|
|
346
|
+
const base64 = await Promise.race([
|
|
347
|
+
this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
|
|
348
|
+
timeoutPromise
|
|
349
|
+
]);
|
|
350
|
+
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
255
351
|
console.log('[HealthCheck] Moving mouse...');
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
271
|
-
} catch (error) {
|
|
272
|
-
console.error(`[HealthCheck] Mouse move failed: ${error}`);
|
|
273
|
-
process.exit(1);
|
|
274
|
-
}
|
|
352
|
+
external_node_assert_default()(device_libnut, 'libnut not initialized');
|
|
353
|
+
const startPos = device_libnut.getMousePos();
|
|
354
|
+
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
355
|
+
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
356
|
+
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
357
|
+
const targetX = startPos.x + offsetX;
|
|
358
|
+
const targetY = startPos.y + offsetY;
|
|
359
|
+
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
360
|
+
device_libnut.moveMouse(targetX, targetY);
|
|
361
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
362
|
+
const movedPos = device_libnut.getMousePos();
|
|
363
|
+
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
364
|
+
device_libnut.moveMouse(startPos.x, startPos.y);
|
|
365
|
+
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
275
366
|
console.log('[HealthCheck] Health check passed');
|
|
276
367
|
}
|
|
277
368
|
async screenshotBase64() {
|
|
@@ -301,8 +392,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
301
392
|
const screenSize = device_libnut.getScreenSize();
|
|
302
393
|
return {
|
|
303
394
|
width: screenSize.width,
|
|
304
|
-
height: screenSize.height
|
|
305
|
-
dpr: 1
|
|
395
|
+
height: screenSize.height
|
|
306
396
|
};
|
|
307
397
|
} catch (error) {
|
|
308
398
|
debugDevice(`Failed to get screen size: ${error}`);
|
|
@@ -567,6 +657,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
567
657
|
}
|
|
568
658
|
async destroy() {
|
|
569
659
|
if (this.destroyed) return;
|
|
660
|
+
if (this.xvfbInstance) {
|
|
661
|
+
this.xvfbInstance.stop();
|
|
662
|
+
this.xvfbInstance = void 0;
|
|
663
|
+
}
|
|
664
|
+
if (this.xvfbCleanup) {
|
|
665
|
+
process.removeListener('exit', this.xvfbCleanup);
|
|
666
|
+
process.removeListener('SIGINT', this.xvfbCleanup);
|
|
667
|
+
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
668
|
+
this.xvfbCleanup = void 0;
|
|
669
|
+
}
|
|
570
670
|
this.destroyed = true;
|
|
571
671
|
debugDevice('Computer device destroyed');
|
|
572
672
|
}
|
|
@@ -579,6 +679,8 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
579
679
|
_define_property(this, "displayId", void 0);
|
|
580
680
|
_define_property(this, "description", void 0);
|
|
581
681
|
_define_property(this, "destroyed", false);
|
|
682
|
+
_define_property(this, "xvfbInstance", void 0);
|
|
683
|
+
_define_property(this, "xvfbCleanup", void 0);
|
|
582
684
|
_define_property(this, "useAppleScript", void 0);
|
|
583
685
|
_define_property(this, "uri", void 0);
|
|
584
686
|
this.options = options;
|
|
@@ -674,7 +776,9 @@ exports.ComputerDevice = __webpack_exports__.ComputerDevice;
|
|
|
674
776
|
exports.agentFromComputer = __webpack_exports__.agentFromComputer;
|
|
675
777
|
exports.checkAccessibilityPermission = __webpack_exports__.checkAccessibilityPermission;
|
|
676
778
|
exports.checkComputerEnvironment = __webpack_exports__.checkComputerEnvironment;
|
|
779
|
+
exports.checkXvfbInstalled = __webpack_exports__.checkXvfbInstalled;
|
|
677
780
|
exports.getConnectedDisplays = __webpack_exports__.getConnectedDisplays;
|
|
781
|
+
exports.needsXvfb = __webpack_exports__.needsXvfb;
|
|
678
782
|
exports.overrideAIConfig = __webpack_exports__.overrideAIConfig;
|
|
679
783
|
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
680
784
|
"ComputerAgent",
|
|
@@ -682,7 +786,9 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
|
682
786
|
"agentFromComputer",
|
|
683
787
|
"checkAccessibilityPermission",
|
|
684
788
|
"checkComputerEnvironment",
|
|
789
|
+
"checkXvfbInstalled",
|
|
685
790
|
"getConnectedDisplays",
|
|
791
|
+
"needsXvfb",
|
|
686
792
|
"overrideAIConfig"
|
|
687
793
|
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
688
794
|
Object.defineProperty(exports, '__esModule', {
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -53,6 +53,74 @@ const img_namespaceObject = require("@midscene/shared/img");
|
|
|
53
53
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
54
54
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
55
55
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
56
|
+
const external_node_fs_namespaceObject = require("node:fs");
|
|
57
|
+
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
58
|
+
function checkXvfbInstalled() {
|
|
59
|
+
try {
|
|
60
|
+
(0, external_node_child_process_namespaceObject.execSync)('which Xvfb', {
|
|
61
|
+
stdio: 'ignore'
|
|
62
|
+
});
|
|
63
|
+
return true;
|
|
64
|
+
} catch {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function findAvailableDisplay(startFrom = 99) {
|
|
69
|
+
for(let n = startFrom; n < startFrom + 100; n++)if (!(0, external_node_fs_namespaceObject.existsSync)(`/tmp/.X${n}-lock`)) return n;
|
|
70
|
+
throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
|
|
71
|
+
}
|
|
72
|
+
function startXvfb(options) {
|
|
73
|
+
const resolution = options?.resolution || '1920x1080x24';
|
|
74
|
+
const displayNum = options?.displayNumber ?? findAvailableDisplay();
|
|
75
|
+
const display = `:${displayNum}`;
|
|
76
|
+
return new Promise((resolve, reject)=>{
|
|
77
|
+
debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
|
|
78
|
+
const xvfbProcess = (0, external_node_child_process_namespaceObject.spawn)('Xvfb', [
|
|
79
|
+
display,
|
|
80
|
+
'-screen',
|
|
81
|
+
'0',
|
|
82
|
+
resolution,
|
|
83
|
+
'-ac',
|
|
84
|
+
'-nolisten',
|
|
85
|
+
'tcp'
|
|
86
|
+
], {
|
|
87
|
+
stdio: 'ignore'
|
|
88
|
+
});
|
|
89
|
+
let settled = false;
|
|
90
|
+
xvfbProcess.on('error', (err)=>{
|
|
91
|
+
if (!settled) {
|
|
92
|
+
settled = true;
|
|
93
|
+
reject(new Error(`Failed to start Xvfb: ${err.message}`));
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
xvfbProcess.on('exit', (code)=>{
|
|
97
|
+
if (!settled) {
|
|
98
|
+
settled = true;
|
|
99
|
+
reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
const instance = {
|
|
103
|
+
process: xvfbProcess,
|
|
104
|
+
display,
|
|
105
|
+
stop () {
|
|
106
|
+
try {
|
|
107
|
+
xvfbProcess.kill('SIGTERM');
|
|
108
|
+
} catch {}
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
setTimeout(()=>{
|
|
112
|
+
if (!settled) {
|
|
113
|
+
settled = true;
|
|
114
|
+
debugXvfb(`Xvfb started on display ${display}`);
|
|
115
|
+
resolve(instance);
|
|
116
|
+
}
|
|
117
|
+
}, 500);
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
function needsXvfb(explicitOpt) {
|
|
121
|
+
if ('linux' !== process.platform) return false;
|
|
122
|
+
return true === explicitOpt;
|
|
123
|
+
}
|
|
56
124
|
function _define_property(obj, key, value) {
|
|
57
125
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
58
126
|
value: value,
|
|
@@ -223,53 +291,74 @@ class ComputerDevice {
|
|
|
223
291
|
async connect() {
|
|
224
292
|
debugDevice('Connecting to computer device');
|
|
225
293
|
try {
|
|
294
|
+
const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
|
|
295
|
+
if (needsXvfb(headless)) {
|
|
296
|
+
if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
|
|
297
|
+
this.xvfbInstance = await startXvfb({
|
|
298
|
+
resolution: this.options?.xvfbResolution
|
|
299
|
+
});
|
|
300
|
+
process.env.DISPLAY = this.xvfbInstance.display;
|
|
301
|
+
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
302
|
+
this.xvfbCleanup = ()=>{
|
|
303
|
+
if (this.xvfbInstance) {
|
|
304
|
+
this.xvfbInstance.stop();
|
|
305
|
+
this.xvfbInstance = void 0;
|
|
306
|
+
}
|
|
307
|
+
};
|
|
308
|
+
process.on('exit', this.xvfbCleanup);
|
|
309
|
+
process.on('SIGINT', this.xvfbCleanup);
|
|
310
|
+
process.on('SIGTERM', this.xvfbCleanup);
|
|
311
|
+
}
|
|
226
312
|
libnut = await getLibnut();
|
|
227
313
|
const size = await this.size();
|
|
228
314
|
const displays = await ComputerDevice.listDisplays();
|
|
315
|
+
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
229
316
|
this.description = `
|
|
230
317
|
Type: Computer
|
|
231
318
|
Platform: ${process.platform}
|
|
232
319
|
Display: ${this.displayId || 'Primary'}
|
|
233
320
|
Screen Size: ${size.width}x${size.height}
|
|
234
|
-
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}
|
|
321
|
+
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
235
322
|
`;
|
|
236
323
|
debugDevice('Computer device connected', this.description);
|
|
324
|
+
await this.healthCheck();
|
|
237
325
|
} catch (error) {
|
|
326
|
+
if (this.xvfbInstance) {
|
|
327
|
+
this.xvfbInstance.stop();
|
|
328
|
+
this.xvfbInstance = void 0;
|
|
329
|
+
}
|
|
238
330
|
debugDevice(`Failed to connect: ${error}`);
|
|
239
331
|
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
240
332
|
}
|
|
241
|
-
await this.healthCheck();
|
|
242
333
|
}
|
|
243
334
|
async healthCheck() {
|
|
244
335
|
console.log('[HealthCheck] Starting health check...');
|
|
245
336
|
console.log('[HealthCheck] Taking screenshot...');
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
337
|
+
const screenshotTimeout = 15000;
|
|
338
|
+
let timeoutId;
|
|
339
|
+
const timeoutPromise = new Promise((_, reject)=>{
|
|
340
|
+
timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
|
|
341
|
+
});
|
|
342
|
+
const base64 = await Promise.race([
|
|
343
|
+
this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
|
|
344
|
+
timeoutPromise
|
|
345
|
+
]);
|
|
346
|
+
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
253
347
|
console.log('[HealthCheck] Moving mouse...');
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
269
|
-
} catch (error) {
|
|
270
|
-
console.error(`[HealthCheck] Mouse move failed: ${error}`);
|
|
271
|
-
process.exit(1);
|
|
272
|
-
}
|
|
348
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
349
|
+
const startPos = libnut.getMousePos();
|
|
350
|
+
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
351
|
+
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
352
|
+
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
353
|
+
const targetX = startPos.x + offsetX;
|
|
354
|
+
const targetY = startPos.y + offsetY;
|
|
355
|
+
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
356
|
+
libnut.moveMouse(targetX, targetY);
|
|
357
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
358
|
+
const movedPos = libnut.getMousePos();
|
|
359
|
+
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
360
|
+
libnut.moveMouse(startPos.x, startPos.y);
|
|
361
|
+
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
273
362
|
console.log('[HealthCheck] Health check passed');
|
|
274
363
|
}
|
|
275
364
|
async screenshotBase64() {
|
|
@@ -299,8 +388,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
299
388
|
const screenSize = libnut.getScreenSize();
|
|
300
389
|
return {
|
|
301
390
|
width: screenSize.width,
|
|
302
|
-
height: screenSize.height
|
|
303
|
-
dpr: 1
|
|
391
|
+
height: screenSize.height
|
|
304
392
|
};
|
|
305
393
|
} catch (error) {
|
|
306
394
|
debugDevice(`Failed to get screen size: ${error}`);
|
|
@@ -565,6 +653,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
565
653
|
}
|
|
566
654
|
async destroy() {
|
|
567
655
|
if (this.destroyed) return;
|
|
656
|
+
if (this.xvfbInstance) {
|
|
657
|
+
this.xvfbInstance.stop();
|
|
658
|
+
this.xvfbInstance = void 0;
|
|
659
|
+
}
|
|
660
|
+
if (this.xvfbCleanup) {
|
|
661
|
+
process.removeListener('exit', this.xvfbCleanup);
|
|
662
|
+
process.removeListener('SIGINT', this.xvfbCleanup);
|
|
663
|
+
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
664
|
+
this.xvfbCleanup = void 0;
|
|
665
|
+
}
|
|
568
666
|
this.destroyed = true;
|
|
569
667
|
debugDevice('Computer device destroyed');
|
|
570
668
|
}
|
|
@@ -577,6 +675,8 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
577
675
|
_define_property(this, "displayId", void 0);
|
|
578
676
|
_define_property(this, "description", void 0);
|
|
579
677
|
_define_property(this, "destroyed", false);
|
|
678
|
+
_define_property(this, "xvfbInstance", void 0);
|
|
679
|
+
_define_property(this, "xvfbCleanup", void 0);
|
|
580
680
|
_define_property(this, "useAppleScript", void 0);
|
|
581
681
|
_define_property(this, "uri", void 0);
|
|
582
682
|
this.options = options;
|
|
@@ -605,7 +705,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
605
705
|
createTemporaryDevice() {
|
|
606
706
|
return new ComputerDevice({});
|
|
607
707
|
}
|
|
608
|
-
async ensureAgent(displayId) {
|
|
708
|
+
async ensureAgent(displayId, headless) {
|
|
609
709
|
if (this.agent && displayId) {
|
|
610
710
|
try {
|
|
611
711
|
await this.agent.destroy?.();
|
|
@@ -616,10 +716,15 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
616
716
|
}
|
|
617
717
|
if (this.agent) return this.agent;
|
|
618
718
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
619
|
-
const opts =
|
|
620
|
-
displayId
|
|
621
|
-
|
|
622
|
-
|
|
719
|
+
const opts = {
|
|
720
|
+
...displayId ? {
|
|
721
|
+
displayId
|
|
722
|
+
} : {},
|
|
723
|
+
...void 0 !== headless ? {
|
|
724
|
+
headless
|
|
725
|
+
} : {}
|
|
726
|
+
};
|
|
727
|
+
const agent = await agentFromComputer(Object.keys(opts).length > 0 ? opts : void 0);
|
|
623
728
|
this.agent = agent;
|
|
624
729
|
return agent;
|
|
625
730
|
}
|
|
@@ -629,10 +734,11 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
629
734
|
name: 'computer_connect',
|
|
630
735
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
631
736
|
schema: {
|
|
632
|
-
displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)')
|
|
737
|
+
displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
738
|
+
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
633
739
|
},
|
|
634
|
-
handler: async ({ displayId })=>{
|
|
635
|
-
const agent = await this.ensureAgent(displayId);
|
|
740
|
+
handler: async ({ displayId, headless })=>{
|
|
741
|
+
const agent = await this.ensureAgent(displayId, headless);
|
|
636
742
|
const screenshot = await agent.interface.screenshotBase64();
|
|
637
743
|
return {
|
|
638
744
|
content: [
|
package/dist/types/index.d.ts
CHANGED
|
@@ -30,6 +30,11 @@ export declare function checkAccessibilityPermission(promptIfNeeded?: boolean):
|
|
|
30
30
|
*/
|
|
31
31
|
export declare function checkComputerEnvironment(): Promise<EnvironmentCheck>;
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Check if Xvfb is installed on the system
|
|
35
|
+
*/
|
|
36
|
+
export declare function checkXvfbInstalled(): boolean;
|
|
37
|
+
|
|
33
38
|
export declare class ComputerAgent extends Agent<ComputerDevice> {
|
|
34
39
|
}
|
|
35
40
|
|
|
@@ -41,6 +46,8 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
41
46
|
private displayId?;
|
|
42
47
|
private description?;
|
|
43
48
|
private destroyed;
|
|
49
|
+
private xvfbInstance?;
|
|
50
|
+
private xvfbCleanup?;
|
|
44
51
|
/**
|
|
45
52
|
* On macOS, use AppleScript for keyboard operations by default
|
|
46
53
|
* to avoid focus issues with system overlays (e.g. Spotlight).
|
|
@@ -86,6 +93,17 @@ export declare interface ComputerDeviceOpt {
|
|
|
86
93
|
* - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps)
|
|
87
94
|
*/
|
|
88
95
|
keyboardDriver?: 'applescript' | 'libnut';
|
|
96
|
+
/**
|
|
97
|
+
* Headless mode via Xvfb (Linux only).
|
|
98
|
+
* - true: start Xvfb virtual display
|
|
99
|
+
* - false/undefined: do not start Xvfb
|
|
100
|
+
* Can also be set via MIDSCENE_COMPUTER_HEADLESS_LINUX=true environment variable.
|
|
101
|
+
*/
|
|
102
|
+
headless?: boolean;
|
|
103
|
+
/**
|
|
104
|
+
* Resolution for Xvfb virtual display (default '1920x1080x24')
|
|
105
|
+
*/
|
|
106
|
+
xvfbResolution?: string;
|
|
89
107
|
}
|
|
90
108
|
|
|
91
109
|
export declare interface DisplayInfo {
|
|
@@ -106,6 +124,13 @@ declare interface EnvironmentCheck {
|
|
|
106
124
|
*/
|
|
107
125
|
export declare function getConnectedDisplays(): Promise<DisplayInfo[]>;
|
|
108
126
|
|
|
127
|
+
/**
|
|
128
|
+
* Determine whether Xvfb is needed.
|
|
129
|
+
* Only starts when explicitly requested via `headless: true`.
|
|
130
|
+
* Non-Linux platforms always return false.
|
|
131
|
+
*/
|
|
132
|
+
export declare function needsXvfb(explicitOpt?: boolean): boolean;
|
|
133
|
+
|
|
109
134
|
export { overrideAIConfig }
|
|
110
135
|
|
|
111
136
|
export { }
|
|
@@ -19,6 +19,8 @@ declare class ComputerDevice implements AbstractInterface {
|
|
|
19
19
|
private displayId?;
|
|
20
20
|
private description?;
|
|
21
21
|
private destroyed;
|
|
22
|
+
private xvfbInstance?;
|
|
23
|
+
private xvfbCleanup?;
|
|
22
24
|
/**
|
|
23
25
|
* On macOS, use AppleScript for keyboard operations by default
|
|
24
26
|
* to avoid focus issues with system overlays (e.g. Spotlight).
|
|
@@ -64,6 +66,17 @@ declare interface ComputerDeviceOpt {
|
|
|
64
66
|
* - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps)
|
|
65
67
|
*/
|
|
66
68
|
keyboardDriver?: 'applescript' | 'libnut';
|
|
69
|
+
/**
|
|
70
|
+
* Headless mode via Xvfb (Linux only).
|
|
71
|
+
* - true: start Xvfb virtual display
|
|
72
|
+
* - false/undefined: do not start Xvfb
|
|
73
|
+
* Can also be set via MIDSCENE_COMPUTER_HEADLESS_LINUX=true environment variable.
|
|
74
|
+
*/
|
|
75
|
+
headless?: boolean;
|
|
76
|
+
/**
|
|
77
|
+
* Resolution for Xvfb virtual display (default '1920x1080x24')
|
|
78
|
+
*/
|
|
79
|
+
xvfbResolution?: string;
|
|
67
80
|
}
|
|
68
81
|
|
|
69
82
|
/**
|
|
@@ -81,7 +94,7 @@ export declare class ComputerMCPServer extends BaseMCPServer {
|
|
|
81
94
|
*/
|
|
82
95
|
declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent> {
|
|
83
96
|
protected createTemporaryDevice(): ComputerDevice;
|
|
84
|
-
protected ensureAgent(displayId?: string): Promise<ComputerAgent>;
|
|
97
|
+
protected ensureAgent(displayId?: string, headless?: boolean): Promise<ComputerAgent>;
|
|
85
98
|
/**
|
|
86
99
|
* Provide Computer-specific platform tools
|
|
87
100
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.7",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -36,8 +36,8 @@
|
|
|
36
36
|
"@computer-use/libnut": "^4.2.0",
|
|
37
37
|
"clipboardy": "^4.0.0",
|
|
38
38
|
"screenshot-desktop": "^1.15.3",
|
|
39
|
-
"@midscene/
|
|
40
|
-
"@midscene/
|
|
39
|
+
"@midscene/shared": "1.4.7",
|
|
40
|
+
"@midscene/core": "1.4.7"
|
|
41
41
|
},
|
|
42
42
|
"optionalDependencies": {
|
|
43
43
|
"node-mac-permissions": "2.5.0"
|