@midscene/computer 1.2.1-beta-20260112081017.0

package/dist/lib/mcp-server.js

@@ -0,0 +1,559 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    mcpKitForAgent: ()=>mcpKitForAgent,
+    mcpServerForAgent: ()=>mcpServerForAgent,
+    ComputerMCPServer: ()=>ComputerMCPServer
+});
+const mcp_namespaceObject = require("@midscene/shared/mcp");
+const agent_namespaceObject = require("@midscene/core/agent");
+const external_node_assert_namespaceObject = require("node:assert");
+var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
+const core_namespaceObject = require("@midscene/core");
+const device_namespaceObject = require("@midscene/core/device");
+const utils_namespaceObject = require("@midscene/core/utils");
+const img_namespaceObject = require("@midscene/shared/img");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
+var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const SMOOTH_MOVE_STEPS_TAP = 8;
+const SMOOTH_MOVE_STEPS_HOVER = 10;
+const SMOOTH_MOVE_DELAY_TAP = 8;
+const SMOOTH_MOVE_DELAY_HOVER = 10;
+const HOVER_EFFECT_WAIT = 300;
+const CLICK_HOLD_DURATION = 50;
+const INPUT_FOCUS_DELAY = 300;
+const INPUT_CLEAR_DELAY = 150;
+const SCROLL_REPEAT_COUNT = 10;
+const SCROLL_STEP_DELAY = 100;
+const SCROLL_COMPLETE_DELAY = 500;
+let libnut = null;
+let libnutLoadError = null;
+async function getLibnut() {
+    if (libnut) return libnut;
+    if (libnutLoadError) throw libnutLoadError;
+    try {
+        const libnutModule = await import("@computer-use/libnut/dist/import_libnut");
+        libnut = libnutModule.libnut;
+        if (!libnut) throw new Error('libnut module loaded but libnut object is undefined');
+        return libnut;
+    } catch (error) {
+        libnutLoadError = error;
+        throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
+    }
+}
+const debugDevice = (0, logger_namespaceObject.getDebug)('computer:device');
+async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
+    external_node_assert_default()(libnut, 'libnut not initialized');
+    const currentPos = libnut.getMousePos();
+    for(let i = 1; i <= steps; i++){
+        const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
+        const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
+        libnut.moveMouse(stepX, stepY);
+        await (0, utils_namespaceObject.sleep)(stepDelay);
+    }
+}
+const KEY_NAME_MAP = {
+    windows: 'win',
+    win: 'win',
+    ctrl: 'control',
+    esc: 'escape',
+    del: 'delete',
+    ins: 'insert',
+    pgup: 'pageup',
+    pgdn: 'pagedown',
+    arrowup: 'up',
+    arrowdown: 'down',
+    arrowleft: 'left',
+    arrowright: 'right',
+    volumedown: 'audio_vol_down',
+    volumeup: 'audio_vol_up',
+    mediavolumedown: 'audio_vol_down',
+    mediavolumeup: 'audio_vol_up',
+    mute: 'audio_mute',
+    mediamute: 'audio_mute',
+    mediaplay: 'audio_play',
+    mediapause: 'audio_pause',
+    mediaplaypause: 'audio_play',
+    mediastop: 'audio_stop',
+    medianexttrack: 'audio_next',
+    mediaprevioustrack: 'audio_prev',
+    medianext: 'audio_next',
+    mediaprev: 'audio_prev'
+};
+const PRIMARY_KEY_MAP = {
+    command: 'cmd',
+    cmd: 'cmd',
+    meta: 'meta',
+    control: 'control',
+    ctrl: 'control',
+    shift: 'shift',
+    alt: 'alt',
+    option: 'alt'
+};
+function normalizeKeyName(key) {
+    const lowerKey = key.toLowerCase();
+    return KEY_NAME_MAP[lowerKey] || lowerKey;
+}
+function normalizePrimaryKey(key) {
+    const lowerKey = key.toLowerCase();
+    if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
+    return KEY_NAME_MAP[lowerKey] || lowerKey;
+}
+class ComputerDevice {
+    describe() {
+        return this.description || 'Computer Device';
+    }
+    static async listDisplays() {
+        try {
+            const displays = await external_screenshot_desktop_default().listDisplays();
+            return displays.map((d)=>({
+                    id: String(d.id),
+                    name: d.name || `Display ${d.id}`,
+                    primary: d.primary || false
+                }));
+        } catch (error) {
+            debugDevice(`Failed to list displays: ${error}`);
+            return [];
+        }
+    }
+    async connect() {
+        debugDevice('Connecting to computer device');
+        try {
+            libnut = await getLibnut();
+            const size = await this.size();
+            const displays = await ComputerDevice.listDisplays();
+            this.description = `
+Type: Computer
+Platform: ${process.platform}
+Display: ${this.displayId || 'Primary'}
+Screen Size: ${size.width}x${size.height}
+Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}
+`;
+            debugDevice('Computer device connected', this.description);
+        } catch (error) {
+            debugDevice(`Failed to connect: ${error}`);
+            throw new Error(`Unable to connect to computer device: ${error}`);
+        }
+    }
+    async screenshotBase64() {
+        debugDevice('Taking screenshot', {
+            displayId: this.displayId
+        });
+        try {
+            const options = {
+                format: 'png'
+            };
+            if (void 0 !== this.displayId) if ('darwin' === process.platform) {
+                const screenIndex = Number(this.displayId);
+                if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
+            } else options.screen = this.displayId;
+            debugDevice('Screenshot options', options);
+            const buffer = await external_screenshot_desktop_default()(options);
+            return (0, img_namespaceObject.createImgBase64ByFormat)('png', buffer.toString('base64'));
+        } catch (error) {
+            debugDevice(`Screenshot failed: ${error}`);
+            throw new Error(`Failed to take screenshot: ${error}`);
+        }
+    }
+    async size() {
+        external_node_assert_default()(libnut, 'libnut not initialized');
+        try {
+            const screenSize = libnut.getScreenSize();
+            return {
+                width: screenSize.width,
+                height: screenSize.height,
+                dpr: 1
+            };
+        } catch (error) {
+            debugDevice(`Failed to get screen size: ${error}`);
+            throw new Error(`Failed to get screen size: ${error}`);
+        }
+    }
+    actionSpace() {
+        const defaultActions = [
+            (0, device_namespaceObject.defineActionTap)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                const element = param.locate;
+                external_node_assert_default()(element, 'Element not found, cannot tap');
+                const [x, y] = element.center;
+                const targetX = Math.round(x);
+                const targetY = Math.round(y);
+                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
+                libnut.mouseToggle('down', 'left');
+                await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
+                libnut.mouseToggle('up', 'left');
+            }),
+            (0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                const element = param.locate;
+                external_node_assert_default()(element, 'Element not found, cannot double click');
+                const [x, y] = element.center;
+                libnut.moveMouse(Math.round(x), Math.round(y));
+                libnut.mouseClick('left', true);
+            }),
+            (0, device_namespaceObject.defineActionRightClick)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                const element = param.locate;
+                external_node_assert_default()(element, 'Element not found, cannot right click');
+                const [x, y] = element.center;
+                libnut.moveMouse(Math.round(x), Math.round(y));
+                libnut.mouseClick('right');
+            }),
+            (0, device_namespaceObject.defineActionHover)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                const element = param.locate;
+                external_node_assert_default()(element, 'Element not found, cannot hover');
+                const [x, y] = element.center;
+                const targetX = Math.round(x);
+                const targetY = Math.round(y);
+                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_HOVER, SMOOTH_MOVE_DELAY_HOVER);
+                await (0, utils_namespaceObject.sleep)(HOVER_EFFECT_WAIT);
+            }),
+            (0, device_namespaceObject.defineAction)({
+                name: 'Input',
+                description: 'Input text into the input field',
+                interfaceAlias: 'aiInput',
+                paramSchema: core_namespaceObject.z.object({
+                    value: core_namespaceObject.z.string().describe('The text to input'),
+                    mode: core_namespaceObject.z["enum"]([
+                        'replace',
+                        'clear',
+                        'append'
+                    ]).default('replace').optional().describe('Input mode: replace, clear, or append'),
+                    locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
+                }),
+                call: async (param)=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    const element = param.locate;
+                    if (element && 'append' !== param.mode) {
+                        const [x, y] = element.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
+                        const modifier = 'darwin' === process.platform ? 'command' : 'control';
+                        libnut.keyTap('a', [
+                            modifier
+                        ]);
+                        await (0, utils_namespaceObject.sleep)(50);
+                        libnut.keyTap('backspace');
+                        await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
+                    }
+                    if ('clear' === param.mode) return;
+                    if (!param.value) return;
+                    libnut.typeString(param.value);
+                }
+            }),
+            (0, device_namespaceObject.defineActionScroll)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                if (param.locate) {
+                    const element = param.locate;
+                    const [x, y] = element.center;
+                    libnut.moveMouse(Math.round(x), Math.round(y));
+                }
+                const scrollType = param?.scrollType;
+                const scrollToEdgeActions = {
+                    scrollToTop: [
+                        0,
+                        10
+                    ],
+                    scrollToBottom: [
+                        0,
+                        -10
+                    ],
+                    scrollToLeft: [
+                        -10,
+                        0
+                    ],
+                    scrollToRight: [
+                        10,
+                        0
+                    ]
+                };
+                const edgeAction = scrollToEdgeActions[scrollType || ''];
+                if (edgeAction) {
+                    const [dx, dy] = edgeAction;
+                    for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
+                        libnut.scrollMouse(dx, dy);
+                        await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
+                    }
+                    return;
+                }
+                if ('singleAction' === scrollType || !scrollType) {
+                    const distance = param?.distance || 500;
+                    const ticks = Math.ceil(distance / 100);
+                    const direction = param?.direction || 'down';
+                    const directionMap = {
+                        up: [
+                            0,
+                            ticks
+                        ],
+                        down: [
+                            0,
+                            -ticks
+                        ],
+                        left: [
+                            -ticks,
+                            0
+                        ],
+                        right: [
+                            ticks,
+                            0
+                        ]
+                    };
+                    const [dx, dy] = directionMap[direction] || [
+                        0,
+                        -ticks
+                    ];
+                    libnut.scrollMouse(dx, dy);
+                    await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
+                    return;
+                }
+                throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
+            }),
+            (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                if (param.locate) {
+                    const [x, y] = param.locate.center;
+                    libnut.moveMouse(Math.round(x), Math.round(y));
+                    libnut.mouseClick('left');
+                    await (0, utils_namespaceObject.sleep)(50);
+                }
+                const keys = param.keyName.split('+');
+                const modifiers = keys.slice(0, -1).map(normalizeKeyName);
+                const key = normalizePrimaryKey(keys[keys.length - 1]);
+                debugDevice('KeyboardPress', {
+                    original: param.keyName,
+                    key,
+                    modifiers
+                });
+                if (modifiers.length > 0) libnut.keyTap(key, modifiers);
+                else libnut.keyTap(key);
+            }),
+            (0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                const from = param.from;
+                const to = param.to;
+                external_node_assert_default()(from, 'missing "from" param for drag and drop');
+                external_node_assert_default()(to, 'missing "to" param for drag and drop');
+                const [fromX, fromY] = from.center;
+                const [toX, toY] = to.center;
+                libnut.moveMouse(Math.round(fromX), Math.round(fromY));
+                libnut.mouseToggle('down', 'left');
+                await (0, utils_namespaceObject.sleep)(100);
+                libnut.moveMouse(Math.round(toX), Math.round(toY));
+                await (0, utils_namespaceObject.sleep)(100);
+                libnut.mouseToggle('up', 'left');
+            }),
+            (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
+                external_node_assert_default()(libnut, 'libnut not initialized');
+                const element = param.locate;
+                external_node_assert_default()(element, 'Element not found, cannot clear input');
+                const [x, y] = element.center;
+                libnut.moveMouse(Math.round(x), Math.round(y));
+                libnut.mouseClick('left');
+                await (0, utils_namespaceObject.sleep)(100);
+                const modifier = 'darwin' === process.platform ? 'command' : 'control';
+                libnut.keyTap('a', [
+                    modifier
+                ]);
+                libnut.keyTap('backspace');
+                await (0, utils_namespaceObject.sleep)(50);
+            })
+        ];
+        const platformActions = Object.values(createPlatformActions());
+        const customActions = this.options?.customActions || [];
+        return [
+            ...defaultActions,
+            ...platformActions,
+            ...customActions
+        ];
+    }
+    async destroy() {
+        if (this.destroyed) return;
+        this.destroyed = true;
+        debugDevice('Computer device destroyed');
+    }
+    async url() {
+        return '';
+    }
+    constructor(options){
+        _define_property(this, "interfaceType", 'computer');
+        _define_property(this, "options", void 0);
+        _define_property(this, "displayId", void 0);
+        _define_property(this, "description", void 0);
+        _define_property(this, "destroyed", false);
+        _define_property(this, "uri", void 0);
+        this.options = options;
+        this.displayId = options?.displayId;
+    }
+}
+function createPlatformActions() {
+    return {
+        ListDisplays: (0, device_namespaceObject.defineAction)({
+            name: 'ListDisplays',
+            description: 'List all available displays/monitors',
+            call: async ()=>await ComputerDevice.listDisplays()
+        })
+    };
+}
+class ComputerAgent extends agent_namespaceObject.Agent {
+}
+async function agentFromComputer(opts) {
+    const device = new ComputerDevice(opts || {});
+    await device.connect();
+    return new ComputerAgent(device, opts);
+}
+const debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
+class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new ComputerDevice({});
+    }
+    async ensureAgent(displayId) {
+        if (this.agent && displayId) {
+            try {
+                await this.agent.destroy?.();
+            } catch (error) {
+                debug('Failed to destroy agent during cleanup:', error);
+            }
+            this.agent = void 0;
+        }
+        if (this.agent) return this.agent;
+        debug('Creating Computer agent with displayId:', displayId || 'primary');
+        const opts = displayId ? {
+            displayId
+        } : void 0;
+        const agent = await agentFromComputer(opts);
+        this.agent = agent;
+        return agent;
+    }
+    preparePlatformTools() {
+        return [
+            {
+                name: 'computer_connect',
+                description: 'Connect to computer desktop. If displayId not provided, uses the primary display.',
+                schema: {
+                    displayId: core_namespaceObject.z.string().optional().describe('Display ID (from list_displays)')
+                },
+                handler: async ({ displayId })=>{
+                    const agent = await this.ensureAgent(displayId);
+                    const screenshot = await agent.interface.screenshotBase64();
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
+                            },
+                            ...this.buildScreenshotContent(screenshot)
+                        ]
+                    };
+                }
+            },
+            {
+                name: 'computer_disconnect',
+                description: 'Disconnect from computer and release resources',
+                schema: {},
+                handler: this.createDisconnectHandler('computer')
+            },
+            {
+                name: 'computer_list_displays',
+                description: 'List all available displays/monitors',
+                schema: {},
+                handler: async ()=>{
+                    const displays = await ComputerDevice.listDisplays();
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
+                            }
+                        ]
+                    };
+                }
+            }
+        ];
+    }
+}
+class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
+    createToolsManager() {
+        return new ComputerMidsceneTools();
+    }
+    constructor(toolsManager){
+        super({
+            name: '@midscene/computer-mcp',
+            version: __VERSION__,
+            description: 'Control the computer desktop using natural language commands'
+        }, toolsManager);
+    }
+}
+function mcpServerForAgent(agent) {
+    return (0, mcp_namespaceObject.createMCPServerLauncher)({
+        agent,
+        platformName: 'Computer',
+        ToolsManagerClass: ComputerMidsceneTools,
+        MCPServerClass: ComputerMCPServer
+    });
+}
+async function mcpKitForAgent(agent) {
+    const toolsManager = new ComputerMidsceneTools();
+    const computerAgent = agent instanceof ComputerAgent ? agent : agent;
+    toolsManager.setAgent(computerAgent);
+    await toolsManager.initTools();
+    return {
+        description: 'Midscene MCP Kit for computer desktop automation',
+        tools: toolsManager.getToolDefinitions()
+    };
+}
+exports.ComputerMCPServer = __webpack_exports__.ComputerMCPServer;
+exports.mcpKitForAgent = __webpack_exports__.mcpKitForAgent;
+exports.mcpServerForAgent = __webpack_exports__.mcpServerForAgent;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "ComputerMCPServer",
+    "mcpKitForAgent",
+    "mcpServerForAgent"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/types/index.d.ts

@@ -0,0 +1,70 @@
+import { AbstractInterface } from '@midscene/core/device';
+import { Agent } from '@midscene/core/agent';
+import { AgentOpt } from '@midscene/core/agent';
+import { DeviceAction } from '@midscene/core';
+import { InterfaceType } from '@midscene/core';
+import { overrideAIConfig } from '@midscene/shared/env';
+import { Size } from '@midscene/core';
+
+/**
+ * Create an Agent from computer
+ */
+export declare function agentFromComputer(opts?: ComputerAgentOpt): Promise<ComputerAgent>;
+
+/**
+ * Check if the computer environment is available
+ */
+export declare function checkComputerEnvironment(): Promise<EnvironmentCheck>;
+
+export declare class ComputerAgent extends Agent<ComputerDevice> {
+}
+
+export declare type ComputerAgentOpt = AgentOpt & ComputerDeviceOpt;
+
+export declare class ComputerDevice implements AbstractInterface {
+    interfaceType: InterfaceType;
+    private options?;
+    private displayId?;
+    private description?;
+    private destroyed;
+    uri?: string;
+    constructor(options?: ComputerDeviceOpt);
+    describe(): string;
+    /**
+     * Get all available displays
+     */
+    static listDisplays(): Promise<DisplayInfo[]>;
+    connect(): Promise<void>;
+    screenshotBase64(): Promise<string>;
+    size(): Promise<Size>;
+    actionSpace(): DeviceAction<any>[];
+    destroy(): Promise<void>;
+    url(): Promise<string>;
+}
+
+export declare interface ComputerDeviceOpt {
+    displayId?: string;
+    customActions?: DeviceAction<any>[];
+}
+
+export declare interface DisplayInfo {
+    id: string;
+    name: string;
+    primary?: boolean;
+}
+
+declare interface EnvironmentCheck {
+    available: boolean;
+    error?: string;
+    platform: string;
+    displays: number;
+}
+
+/**
+ * Get all connected displays
+ */
+export declare function getConnectedDisplays(): Promise<DisplayInfo[]>;
+
+export { overrideAIConfig }
+
+export { }