@midscene/computer 1.5.3 → 1.5.4-beta-20260310030546.0

package/dist/es/index.mjs

@@ -9,6 +9,7 @@ import { getDebug } from "@midscene/shared/logger";
 import screenshot_desktop from "screenshot-desktop";
 import { existsSync } from "node:fs";
 import { Agent } from "@midscene/core/agent";
+import { BaseMidsceneTools } from "@midscene/shared/mcp";
 import { overrideAIConfig } from "@midscene/shared/env";
 const debugXvfb = getDebug('computer:xvfb');
 function checkXvfbInstalled() {
@@ -679,6 +680,82 @@ async function agentFromComputer(opts) {
     await device.connect();
     return new ComputerAgent(device, opts);
 }
+const debug = getDebug('mcp:computer-tools');
+class ComputerMidsceneTools extends BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new ComputerDevice({});
+    }
+    async ensureAgent(displayId, headless) {
+        if (this.agent && displayId) {
+            try {
+                await this.agent.destroy?.();
+            } catch (error) {
+                debug('Failed to destroy agent during cleanup:', error);
+            }
+            this.agent = void 0;
+        }
+        if (this.agent) return this.agent;
+        debug('Creating Computer agent with displayId:', displayId || 'primary');
+        const opts = {
+            ...displayId ? {
+                displayId
+            } : {},
+            ...void 0 !== headless ? {
+                headless
+            } : {}
+        };
+        const agent = await agentFromComputer(Object.keys(opts).length > 0 ? opts : void 0);
+        this.agent = agent;
+        return agent;
+    }
+    preparePlatformTools() {
+        return [
+            {
+                name: 'computer_connect',
+                description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
+                schema: {
+                    displayId: z.string().optional().describe('Display ID (from computer_list_displays)'),
+                    headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
+                },
+                handler: async ({ displayId, headless })=>{
+                    const agent = await this.ensureAgent(displayId, headless);
+                    const screenshot = await agent.interface.screenshotBase64();
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
+                            },
+                            ...this.buildScreenshotContent(screenshot)
+                        ]
+                    };
+                }
+            },
+            {
+                name: 'computer_disconnect',
+                description: 'Disconnect from computer and release resources',
+                schema: {},
+                handler: this.createDisconnectHandler('computer')
+            },
+            {
+                name: 'computer_list_displays',
+                description: 'List all available displays/monitors',
+                schema: {},
+                handler: async ()=>{
+                    const displays = await ComputerDevice.listDisplays();
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
+                            }
+                        ]
+                    };
+                }
+            }
+        ];
+    }
+}
 function checkAccessibilityPermission(promptIfNeeded = false) {
     if ('darwin' !== process.platform) return {
         hasPermission: true,
@@ -744,4 +821,4 @@ async function checkComputerEnvironment() {
 async function getConnectedDisplays() {
     return ComputerDevice.listDisplays();
 }
-export { ComputerAgent, ComputerDevice, agentFromComputer, checkAccessibilityPermission, checkComputerEnvironment, checkXvfbInstalled, getConnectedDisplays, needsXvfb, overrideAIConfig };
+export { ComputerAgent, ComputerDevice, ComputerMidsceneTools, agentFromComputer, checkAccessibilityPermission, checkComputerEnvironment, checkXvfbInstalled, getConnectedDisplays, needsXvfb, overrideAIConfig };

package/dist/lib/index.js

@@ -42,6 +42,7 @@ __webpack_require__.d(__webpack_exports__, {
     agentFromComputer: ()=>agentFromComputer,
     needsXvfb: ()=>needsXvfb,
     overrideAIConfig: ()=>env_namespaceObject.overrideAIConfig,
+    ComputerMidsceneTools: ()=>ComputerMidsceneTools,
     getConnectedDisplays: ()=>getConnectedDisplays,
     ComputerDevice: ()=>ComputerDevice,
     ComputerAgent: ()=>ComputerAgent
@@ -728,6 +729,83 @@ async function agentFromComputer(opts) {
     await device.connect();
     return new ComputerAgent(device, opts);
 }
+const mcp_namespaceObject = require("@midscene/shared/mcp");
+const debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
+class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new ComputerDevice({});
+    }
+    async ensureAgent(displayId, headless) {
+        if (this.agent && displayId) {
+            try {
+                await this.agent.destroy?.();
+            } catch (error) {
+                debug('Failed to destroy agent during cleanup:', error);
+            }
+            this.agent = void 0;
+        }
+        if (this.agent) return this.agent;
+        debug('Creating Computer agent with displayId:', displayId || 'primary');
+        const opts = {
+            ...displayId ? {
+                displayId
+            } : {},
+            ...void 0 !== headless ? {
+                headless
+            } : {}
+        };
+        const agent = await agentFromComputer(Object.keys(opts).length > 0 ? opts : void 0);
+        this.agent = agent;
+        return agent;
+    }
+    preparePlatformTools() {
+        return [
+            {
+                name: 'computer_connect',
+                description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
+                schema: {
+                    displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
+                    headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
+                },
+                handler: async ({ displayId, headless })=>{
+                    const agent = await this.ensureAgent(displayId, headless);
+                    const screenshot = await agent.interface.screenshotBase64();
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
+                            },
+                            ...this.buildScreenshotContent(screenshot)
+                        ]
+                    };
+                }
+            },
+            {
+                name: 'computer_disconnect',
+                description: 'Disconnect from computer and release resources',
+                schema: {},
+                handler: this.createDisconnectHandler('computer')
+            },
+            {
+                name: 'computer_list_displays',
+                description: 'List all available displays/monitors',
+                schema: {},
+                handler: async ()=>{
+                    const displays = await ComputerDevice.listDisplays();
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
+                            }
+                        ]
+                    };
+                }
+            }
+        ];
+    }
+}
 const env_namespaceObject = require("@midscene/shared/env");
 function checkAccessibilityPermission(promptIfNeeded = false) {
     if ('darwin' !== process.platform) return {
@@ -796,6 +874,7 @@ async function getConnectedDisplays() {
 }
 exports.ComputerAgent = __webpack_exports__.ComputerAgent;
 exports.ComputerDevice = __webpack_exports__.ComputerDevice;
+exports.ComputerMidsceneTools = __webpack_exports__.ComputerMidsceneTools;
 exports.agentFromComputer = __webpack_exports__.agentFromComputer;
 exports.checkAccessibilityPermission = __webpack_exports__.checkAccessibilityPermission;
 exports.checkComputerEnvironment = __webpack_exports__.checkComputerEnvironment;
@@ -806,6 +885,7 @@ exports.overrideAIConfig = __webpack_exports__.overrideAIConfig;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "ComputerAgent",
     "ComputerDevice",
+    "ComputerMidsceneTools",
     "agentFromComputer",
     "checkAccessibilityPermission",
     "checkComputerEnvironment",

package/dist/types/index.d.ts

@@ -1,10 +1,12 @@
 import { AbstractInterface } from '@midscene/core/device';
 import { Agent } from '@midscene/core/agent';
 import { AgentOpt } from '@midscene/core/agent';
+import { BaseMidsceneTools } from '@midscene/shared/mcp';
 import { DeviceAction } from '@midscene/core';
 import { InterfaceType } from '@midscene/core';
 import { overrideAIConfig } from '@midscene/shared/env';
 import { Size } from '@midscene/core';
+import { ToolDefinition } from '@midscene/shared/mcp';
 
 declare interface AccessibilityCheckResult {
     hasPermission: boolean;
@@ -111,6 +113,19 @@ export declare interface ComputerDeviceOpt {
     xvfbResolution?: string;
 }
 
+/**
+ * Computer-specific tools manager
+ * Extends BaseMidsceneTools to provide desktop automation tools
+ */
+export declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent> {
+    protected createTemporaryDevice(): ComputerDevice;
+    protected ensureAgent(displayId?: string, headless?: boolean): Promise<ComputerAgent>;
+    /**
+     * Provide Computer-specific platform tools
+     */
+    protected preparePlatformTools(): ToolDefinition[];
+}
+
 export declare interface DisplayInfo {
     id: string;
     name: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/computer",
-  "version": "1.5.3",
+  "version": "1.5.4-beta-20260310030546.0",
   "description": "Midscene.js Computer Desktop Automation",
   "license": "MIT",
   "repository": {
@@ -36,8 +36,8 @@
     "@computer-use/libnut": "^4.2.0",
     "clipboardy": "^4.0.0",
     "screenshot-desktop": "^1.15.3",
-    "@midscene/core": "1.5.3",
-    "@midscene/shared": "1.5.3"
+    "@midscene/core": "1.5.4-beta-20260310030546.0",
+    "@midscene/shared": "1.5.4-beta-20260310030546.0"
   },
   "optionalDependencies": {
     "node-mac-permissions": "2.5.0"