@midscene/computer 1.4.6 → 1.4.7

package/dist/es/cli.mjs

@@ -4,12 +4,80 @@ import { getDebug } from "@midscene/shared/logger";
 import { BaseMidsceneTools } from "@midscene/shared/mcp";
 import { Agent } from "@midscene/core/agent";
 import node_assert from "node:assert";
-import { execSync } from "node:child_process";
+import { execSync, spawn } from "node:child_process";
 import { createRequire } from "node:module";
 import { actionHoverParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
 import { sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import screenshot_desktop from "screenshot-desktop";
+import { existsSync } from "node:fs";
+const debugXvfb = getDebug('computer:xvfb');
+function checkXvfbInstalled() {
+    try {
+        execSync('which Xvfb', {
+            stdio: 'ignore'
+        });
+        return true;
+    } catch  {
+        return false;
+    }
+}
+function findAvailableDisplay(startFrom = 99) {
+    for(let n = startFrom; n < startFrom + 100; n++)if (!existsSync(`/tmp/.X${n}-lock`)) return n;
+    throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
+}
+function startXvfb(options) {
+    const resolution = options?.resolution || '1920x1080x24';
+    const displayNum = options?.displayNumber ?? findAvailableDisplay();
+    const display = `:${displayNum}`;
+    return new Promise((resolve, reject)=>{
+        debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
+        const xvfbProcess = spawn('Xvfb', [
+            display,
+            '-screen',
+            '0',
+            resolution,
+            '-ac',
+            '-nolisten',
+            'tcp'
+        ], {
+            stdio: 'ignore'
+        });
+        let settled = false;
+        xvfbProcess.on('error', (err)=>{
+            if (!settled) {
+                settled = true;
+                reject(new Error(`Failed to start Xvfb: ${err.message}`));
+            }
+        });
+        xvfbProcess.on('exit', (code)=>{
+            if (!settled) {
+                settled = true;
+                reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
+            }
+        });
+        const instance = {
+            process: xvfbProcess,
+            display,
+            stop () {
+                try {
+                    xvfbProcess.kill('SIGTERM');
+                } catch  {}
+            }
+        };
+        setTimeout(()=>{
+            if (!settled) {
+                settled = true;
+                debugXvfb(`Xvfb started on display ${display}`);
+                resolve(instance);
+            }
+        }, 500);
+    });
+}
+function needsXvfb(explicitOpt) {
+    if ('linux' !== process.platform) return false;
+    return true === explicitOpt;
+}
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -180,53 +248,74 @@ class ComputerDevice {
     async connect() {
         debugDevice('Connecting to computer device');
         try {
+            const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
+            if (needsXvfb(headless)) {
+                if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
+                this.xvfbInstance = await startXvfb({
+                    resolution: this.options?.xvfbResolution
+                });
+                process.env.DISPLAY = this.xvfbInstance.display;
+                debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
+                this.xvfbCleanup = ()=>{
+                    if (this.xvfbInstance) {
+                        this.xvfbInstance.stop();
+                        this.xvfbInstance = void 0;
+                    }
+                };
+                process.on('exit', this.xvfbCleanup);
+                process.on('SIGINT', this.xvfbCleanup);
+                process.on('SIGTERM', this.xvfbCleanup);
+            }
             libnut = await getLibnut();
             const size = await this.size();
             const displays = await ComputerDevice.listDisplays();
+            const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
             this.description = `
 Type: Computer
 Platform: ${process.platform}
 Display: ${this.displayId || 'Primary'}
 Screen Size: ${size.width}x${size.height}
-Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}
+Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
 `;
             debugDevice('Computer device connected', this.description);
+            await this.healthCheck();
         } catch (error) {
+            if (this.xvfbInstance) {
+                this.xvfbInstance.stop();
+                this.xvfbInstance = void 0;
+            }
             debugDevice(`Failed to connect: ${error}`);
             throw new Error(`Unable to connect to computer device: ${error}`);
         }
-        await this.healthCheck();
     }
     async healthCheck() {
         console.log('[HealthCheck] Starting health check...');
         console.log('[HealthCheck] Taking screenshot...');
-        try {
-            const base64 = await this.screenshotBase64();
-            console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
-        } catch (error) {
-            console.error(`[HealthCheck] Screenshot failed: ${error}`);
-            process.exit(1);
-        }
+        const screenshotTimeout = 15000;
+        let timeoutId;
+        const timeoutPromise = new Promise((_, reject)=>{
+            timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
+        });
+        const base64 = await Promise.race([
+            this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
+            timeoutPromise
+        ]);
+        console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
         console.log('[HealthCheck] Moving mouse...');
-        try {
-            node_assert(libnut, 'libnut not initialized');
-            const startPos = libnut.getMousePos();
-            console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
-            const offsetX = Math.floor(40 * Math.random()) + 10;
-            const offsetY = Math.floor(40 * Math.random()) + 10;
-            const targetX = startPos.x + offsetX;
-            const targetY = startPos.y + offsetY;
-            console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
-            libnut.moveMouse(targetX, targetY);
-            await sleep(50);
-            const movedPos = libnut.getMousePos();
-            console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
-            libnut.moveMouse(startPos.x, startPos.y);
-            console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
-        } catch (error) {
-            console.error(`[HealthCheck] Mouse move failed: ${error}`);
-            process.exit(1);
-        }
+        node_assert(libnut, 'libnut not initialized');
+        const startPos = libnut.getMousePos();
+        console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
+        const offsetX = Math.floor(40 * Math.random()) + 10;
+        const offsetY = Math.floor(40 * Math.random()) + 10;
+        const targetX = startPos.x + offsetX;
+        const targetY = startPos.y + offsetY;
+        console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
+        libnut.moveMouse(targetX, targetY);
+        await sleep(50);
+        const movedPos = libnut.getMousePos();
+        console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
+        libnut.moveMouse(startPos.x, startPos.y);
+        console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
         console.log('[HealthCheck] Health check passed');
     }
     async screenshotBase64() {
@@ -256,8 +345,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
             const screenSize = libnut.getScreenSize();
             return {
                 width: screenSize.width,
-                height: screenSize.height,
-                dpr: 1
+                height: screenSize.height
             };
         } catch (error) {
             debugDevice(`Failed to get screen size: ${error}`);
@@ -522,6 +610,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async destroy() {
         if (this.destroyed) return;
+        if (this.xvfbInstance) {
+            this.xvfbInstance.stop();
+            this.xvfbInstance = void 0;
+        }
+        if (this.xvfbCleanup) {
+            process.removeListener('exit', this.xvfbCleanup);
+            process.removeListener('SIGINT', this.xvfbCleanup);
+            process.removeListener('SIGTERM', this.xvfbCleanup);
+            this.xvfbCleanup = void 0;
+        }
         this.destroyed = true;
         debugDevice('Computer device destroyed');
     }
@@ -534,6 +632,8 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         _define_property(this, "displayId", void 0);
         _define_property(this, "description", void 0);
         _define_property(this, "destroyed", false);
+        _define_property(this, "xvfbInstance", void 0);
+        _define_property(this, "xvfbCleanup", void 0);
         _define_property(this, "useAppleScript", void 0);
         _define_property(this, "uri", void 0);
         this.options = options;
@@ -562,7 +662,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
     createTemporaryDevice() {
         return new ComputerDevice({});
     }
-    async ensureAgent(displayId) {
+    async ensureAgent(displayId, headless) {
         if (this.agent && displayId) {
             try {
                 await this.agent.destroy?.();
@@ -573,10 +673,15 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
         }
         if (this.agent) return this.agent;
         debug('Creating Computer agent with displayId:', displayId || 'primary');
-        const opts = displayId ? {
-            displayId
-        } : void 0;
-        const agent = await agentFromComputer(opts);
+        const opts = {
+            ...displayId ? {
+                displayId
+            } : {},
+            ...void 0 !== headless ? {
+                headless
+            } : {}
+        };
+        const agent = await agentFromComputer(Object.keys(opts).length > 0 ? opts : void 0);
         this.agent = agent;
         return agent;
     }
@@ -586,10 +691,11 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
                 name: 'computer_connect',
                 description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
                 schema: {
-                    displayId: z.string().optional().describe('Display ID (from computer_list_displays)')
+                    displayId: z.string().optional().describe('Display ID (from computer_list_displays)'),
+                    headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
                 },
-                handler: async ({ displayId })=>{
-                    const agent = await this.ensureAgent(displayId);
+                handler: async ({ displayId, headless })=>{
+                    const agent = await this.ensureAgent(displayId, headless);
                     const screenshot = await agent.interface.screenshotBase64();
                     return {
                         content: [

package/dist/es/index.mjs

@@ -1,5 +1,5 @@
 import node_assert from "node:assert";
-import { execSync } from "node:child_process";
+import { execSync, spawn } from "node:child_process";
 import { createRequire } from "node:module";
 import { getMidsceneLocationSchema, z } from "@midscene/core";
 import { actionHoverParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
@@ -7,8 +7,76 @@ import { sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import screenshot_desktop from "screenshot-desktop";
+import { existsSync } from "node:fs";
 import { Agent } from "@midscene/core/agent";
 import { overrideAIConfig } from "@midscene/shared/env";
+const debugXvfb = getDebug('computer:xvfb');
+function checkXvfbInstalled() {
+    try {
+        execSync('which Xvfb', {
+            stdio: 'ignore'
+        });
+        return true;
+    } catch  {
+        return false;
+    }
+}
+function findAvailableDisplay(startFrom = 99) {
+    for(let n = startFrom; n < startFrom + 100; n++)if (!existsSync(`/tmp/.X${n}-lock`)) return n;
+    throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
+}
+function startXvfb(options) {
+    const resolution = options?.resolution || '1920x1080x24';
+    const displayNum = options?.displayNumber ?? findAvailableDisplay();
+    const display = `:${displayNum}`;
+    return new Promise((resolve, reject)=>{
+        debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
+        const xvfbProcess = spawn('Xvfb', [
+            display,
+            '-screen',
+            '0',
+            resolution,
+            '-ac',
+            '-nolisten',
+            'tcp'
+        ], {
+            stdio: 'ignore'
+        });
+        let settled = false;
+        xvfbProcess.on('error', (err)=>{
+            if (!settled) {
+                settled = true;
+                reject(new Error(`Failed to start Xvfb: ${err.message}`));
+            }
+        });
+        xvfbProcess.on('exit', (code)=>{
+            if (!settled) {
+                settled = true;
+                reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
+            }
+        });
+        const instance = {
+            process: xvfbProcess,
+            display,
+            stop () {
+                try {
+                    xvfbProcess.kill('SIGTERM');
+                } catch  {}
+            }
+        };
+        setTimeout(()=>{
+            if (!settled) {
+                settled = true;
+                debugXvfb(`Xvfb started on display ${display}`);
+                resolve(instance);
+            }
+        }, 500);
+    });
+}
+function needsXvfb(explicitOpt) {
+    if ('linux' !== process.platform) return false;
+    return true === explicitOpt;
+}
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -179,53 +247,74 @@ class ComputerDevice {
     async connect() {
         debugDevice('Connecting to computer device');
         try {
+            const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
+            if (needsXvfb(headless)) {
+                if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
+                this.xvfbInstance = await startXvfb({
+                    resolution: this.options?.xvfbResolution
+                });
+                process.env.DISPLAY = this.xvfbInstance.display;
+                debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
+                this.xvfbCleanup = ()=>{
+                    if (this.xvfbInstance) {
+                        this.xvfbInstance.stop();
+                        this.xvfbInstance = void 0;
+                    }
+                };
+                process.on('exit', this.xvfbCleanup);
+                process.on('SIGINT', this.xvfbCleanup);
+                process.on('SIGTERM', this.xvfbCleanup);
+            }
             device_libnut = await getLibnut();
             const size = await this.size();
             const displays = await ComputerDevice.listDisplays();
+            const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
             this.description = `
 Type: Computer
 Platform: ${process.platform}
 Display: ${this.displayId || 'Primary'}
 Screen Size: ${size.width}x${size.height}
-Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}
+Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
 `;
             debugDevice('Computer device connected', this.description);
+            await this.healthCheck();
         } catch (error) {
+            if (this.xvfbInstance) {
+                this.xvfbInstance.stop();
+                this.xvfbInstance = void 0;
+            }
             debugDevice(`Failed to connect: ${error}`);
             throw new Error(`Unable to connect to computer device: ${error}`);
         }
-        await this.healthCheck();
     }
     async healthCheck() {
         console.log('[HealthCheck] Starting health check...');
         console.log('[HealthCheck] Taking screenshot...');
-        try {
-            const base64 = await this.screenshotBase64();
-            console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
-        } catch (error) {
-            console.error(`[HealthCheck] Screenshot failed: ${error}`);
-            process.exit(1);
-        }
+        const screenshotTimeout = 15000;
+        let timeoutId;
+        const timeoutPromise = new Promise((_, reject)=>{
+            timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
+        });
+        const base64 = await Promise.race([
+            this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
+            timeoutPromise
+        ]);
+        console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
         console.log('[HealthCheck] Moving mouse...');
-        try {
-            node_assert(device_libnut, 'libnut not initialized');
-            const startPos = device_libnut.getMousePos();
-            console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
-            const offsetX = Math.floor(40 * Math.random()) + 10;
-            const offsetY = Math.floor(40 * Math.random()) + 10;
-            const targetX = startPos.x + offsetX;
-            const targetY = startPos.y + offsetY;
-            console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
-            device_libnut.moveMouse(targetX, targetY);
-            await sleep(50);
-            const movedPos = device_libnut.getMousePos();
-            console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
-            device_libnut.moveMouse(startPos.x, startPos.y);
-            console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
-        } catch (error) {
-            console.error(`[HealthCheck] Mouse move failed: ${error}`);
-            process.exit(1);
-        }
+        node_assert(device_libnut, 'libnut not initialized');
+        const startPos = device_libnut.getMousePos();
+        console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
+        const offsetX = Math.floor(40 * Math.random()) + 10;
+        const offsetY = Math.floor(40 * Math.random()) + 10;
+        const targetX = startPos.x + offsetX;
+        const targetY = startPos.y + offsetY;
+        console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
+        device_libnut.moveMouse(targetX, targetY);
+        await sleep(50);
+        const movedPos = device_libnut.getMousePos();
+        console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
+        device_libnut.moveMouse(startPos.x, startPos.y);
+        console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
         console.log('[HealthCheck] Health check passed');
     }
     async screenshotBase64() {
@@ -255,8 +344,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
             const screenSize = device_libnut.getScreenSize();
             return {
                 width: screenSize.width,
-                height: screenSize.height,
-                dpr: 1
+                height: screenSize.height
             };
         } catch (error) {
             debugDevice(`Failed to get screen size: ${error}`);
@@ -521,6 +609,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async destroy() {
         if (this.destroyed) return;
+        if (this.xvfbInstance) {
+            this.xvfbInstance.stop();
+            this.xvfbInstance = void 0;
+        }
+        if (this.xvfbCleanup) {
+            process.removeListener('exit', this.xvfbCleanup);
+            process.removeListener('SIGINT', this.xvfbCleanup);
+            process.removeListener('SIGTERM', this.xvfbCleanup);
+            this.xvfbCleanup = void 0;
+        }
         this.destroyed = true;
         debugDevice('Computer device destroyed');
     }
@@ -533,6 +631,8 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         _define_property(this, "displayId", void 0);
         _define_property(this, "description", void 0);
         _define_property(this, "destroyed", false);
+        _define_property(this, "xvfbInstance", void 0);
+        _define_property(this, "xvfbCleanup", void 0);
         _define_property(this, "useAppleScript", void 0);
         _define_property(this, "uri", void 0);
         this.options = options;
@@ -621,4 +721,4 @@ async function checkComputerEnvironment() {
 async function getConnectedDisplays() {
     return ComputerDevice.listDisplays();
 }
-export { ComputerAgent, ComputerDevice, agentFromComputer, checkAccessibilityPermission, checkComputerEnvironment, getConnectedDisplays, overrideAIConfig };
+export { ComputerAgent, ComputerDevice, agentFromComputer, checkAccessibilityPermission, checkComputerEnvironment, checkXvfbInstalled, getConnectedDisplays, needsXvfb, overrideAIConfig };