@midscene/computer 1.2.1-beta-20260112081017.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,243 @@
+# @midscene/computer
+
+Midscene.js Computer Desktop Automation - AI-powered desktop automation for Windows, macOS, and Linux.
+
+## Features
+
+- 🖥️ **Desktop Automation**: Control mouse, keyboard, and screen
+- 📸 **Screenshot Capture**: Take screenshots of any display
+- 🖱️ **Mouse Operations**: Click, double-click, right-click, hover, drag & drop
+- ⌨️ **Keyboard Input**: Type text, press keys, shortcuts
+- 📜 **Scroll Operations**: Scroll in any direction
+- 🖼️ **Multi-Display Support**: Work with multiple monitors
+- 🤖 **AI-Powered**: Use natural language to control your desktop
+- 🔌 **MCP Server**: Expose capabilities via Model Context Protocol
+
+## Installation
+
+```bash
+npm install @midscene/computer
+# or
+pnpm add @midscene/computer
+```
+
+### Platform Requirements
+
+This package uses native modules for desktop control:
+- `screenshot-desktop`: For capturing screenshots
+- `@computer-use/libnut`: For mouse and keyboard control
+
+These modules require compilation on installation. Make sure you have the necessary build tools:
+
+**macOS**: Install Xcode Command Line Tools
+```bash
+xcode-select --install
+```
+
+**Linux**: Install build essentials and ImageMagick
+```bash
+# Ubuntu/Debian
+sudo apt-get install build-essential libx11-dev libxtst-dev libpng-dev imagemagick
+
+# Fedora/RHEL
+sudo dnf install gcc-c++ libX11-devel libXtst-devel libpng-devel ImageMagick
+
+# Arch
+sudo pacman -S base-devel libx11 libxtst libpng imagemagick
+```
+
+**Note**: ImageMagick is required for screenshot capture on Linux.
+
+**Windows**: Install Windows Build Tools
+```bash
+npm install --global windows-build-tools
+```
+
+## Quick Start
+
+### Basic Usage
+
+```typescript
+import { agentFromComputer } from '@midscene/computer';
+
+// Create an agent
+const agent = await agentFromComputer({
+  aiActionContext: 'You are controlling a desktop computer.',
+});
+
+// Use AI to perform actions
+await agent.aiAct('move mouse to center of screen');
+await agent.aiAct('click on the desktop');
+await agent.aiAct('type "Hello World"');
+
+// Query information
+const screenInfo = await agent.aiQuery(
+  '{width: number, height: number}, get screen resolution',
+);
+
+// Assert conditions
+await agent.aiAssert('There is a desktop visible');
+```
+
+### Multi-Display Support
+
+```typescript
+import { ComputerDevice, agentFromComputer } from '@midscene/computer';
+
+// List all displays
+const displays = await ComputerDevice.listDisplays();
+console.log('Available displays:', displays);
+
+// Connect to a specific display
+const agent = await agentFromComputer({
+  displayId: displays[0].id,
+});
+```
+
+### Environment Check
+
+```typescript
+import { checkComputerEnvironment } from '@midscene/computer';
+
+const env = await checkComputerEnvironment();
+console.log('Platform:', env.platform);
+console.log('Available:', env.available);
+console.log('Displays:', env.displays);
+```
+
+## Available Actions
+
+The ComputerDevice supports the following actions:
+
+- **Tap**: Single click at element center
+- **DoubleClick**: Double click at element center
+- **RightClick**: Right click at element center
+- **Hover**: Move mouse to element center
+- **Input**: Type text with different modes (replace/clear/append)
+- **Scroll**: Scroll in any direction (up/down/left/right)
+- **KeyboardPress**: Press keyboard keys with modifiers
+- **DragAndDrop**: Drag from one element to another
+- **ClearInput**: Clear input field content
+- **ListDisplays**: Get all available displays
+
+## Platform-Specific Shortcuts
+
+### macOS
+- Modifier key: `Cmd` (Command)
+- Open search: `Cmd+Space`
+- Select all: `Cmd+A`
+- Copy: `Cmd+C`
+- Paste: `Cmd+V`
+
+### Windows/Linux
+- Modifier key: `Ctrl` (Control)
+- Open search: `Windows key` or `Super key`
+- Select all: `Ctrl+A`
+- Copy: `Ctrl+C`
+- Paste: `Ctrl+V`
+
+## Testing
+
+### Run Unit Tests
+
+```bash
+pnpm test
+```
+
+### Run AI Tests
+
+```bash
+# Set AI_TEST_TYPE environment variable
+AI_TEST_TYPE=computer pnpm test:ai
+```
+
+Available AI tests:
+- `basic.test.ts`: Basic desktop interactions
+- `multi-display.test.ts`: Multi-display support
+- `web-browser.test.ts`: Browser automation
+- `text-editor.test.ts`: Text editor operations
+
+## MCP Server
+
+Start the MCP server for AI assistant integration:
+
+```typescript
+import { mcpServerForAgent } from '@midscene/computer/mcp-server';
+import { agentFromComputer } from '@midscene/computer';
+
+const agent = await agentFromComputer();
+const { server } = mcpServerForAgent(agent);
+await server.launch();
+```
+
+Available MCP tools:
+- `computer_connect`: Connect to desktop display
+- `computer_list_displays`: List all available displays
+- Plus all standard Midscene tools (aiAct, aiQuery, aiAssert, etc.)
+
+## Architecture
+
+This package follows the same architecture pattern as `@midscene/android` and `@midscene/ios`:
+
+```
+packages/computer/
+├── src/
+│   ├── device.ts        # ComputerDevice - core device implementation
+│   ├── agent.ts         # ComputerAgent - agent wrapper
+│   ├── utils.ts         # Utility functions
+│   ├── mcp-server.ts    # MCP server
+│   └── mcp-tools.ts     # MCP tools definitions
+├── tests/
+│   ├── unit-test/       # Unit tests (no native dependencies)
+│   └── ai/              # AI-powered integration tests
+└── README.md
+```
+
+## API Reference
+
+### ComputerDevice
+
+```typescript
+class ComputerDevice implements AbstractInterface {
+  constructor(options?: ComputerDeviceOpt);
+
+  static listDisplays(): Promise<DisplayInfo[]>;
+
+  async connect(): Promise<void>;
+  async screenshotBase64(): Promise<string>;
+  async size(): Promise<Size>;
+  actionSpace(): DeviceAction<any>[];
+  async destroy(): Promise<void>;
+}
+```
+
+### ComputerAgent
+
+```typescript
+class ComputerAgent extends PageAgent<ComputerDevice> {
+  // Inherits all PageAgent methods
+  async aiAct(action: string): Promise<void>;
+  async aiQuery(query: string): Promise<any>;
+  async aiAssert(assertion: string): Promise<void>;
+  async aiWaitFor(condition: string): Promise<void>;
+}
+```
+
+### Factory Functions
+
+```typescript
+async function agentFromComputer(
+  opts?: ComputerAgentOpt
+): Promise<ComputerAgent>;
+
+async function checkComputerEnvironment(): Promise<EnvironmentCheck>;
+async function getConnectedDisplays(): Promise<DisplayInfo[]>;
+```
+
+## License
+
+MIT
+
+## Contributing
+
+See the main [Midscene.js repository](https://github.com/web-infra-dev/midscene) for contributing guidelines.

package/dist/es/index.mjs

@@ -0,0 +1,438 @@
+import node_assert from "node:assert";
+import { getMidsceneLocationSchema, z } from "@midscene/core";
+import { defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
+import { sleep } from "@midscene/core/utils";
+import { createImgBase64ByFormat } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import screenshot_desktop from "screenshot-desktop";
+import { Agent } from "@midscene/core/agent";
+import { overrideAIConfig } from "@midscene/shared/env";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const SMOOTH_MOVE_STEPS_TAP = 8;
+const SMOOTH_MOVE_STEPS_HOVER = 10;
+const SMOOTH_MOVE_DELAY_TAP = 8;
+const SMOOTH_MOVE_DELAY_HOVER = 10;
+const HOVER_EFFECT_WAIT = 300;
+const CLICK_HOLD_DURATION = 50;
+const INPUT_FOCUS_DELAY = 300;
+const INPUT_CLEAR_DELAY = 150;
+const SCROLL_REPEAT_COUNT = 10;
+const SCROLL_STEP_DELAY = 100;
+const SCROLL_COMPLETE_DELAY = 500;
+let device_libnut = null;
+let libnutLoadError = null;
+async function getLibnut() {
+    if (device_libnut) return device_libnut;
+    if (libnutLoadError) throw libnutLoadError;
+    try {
+        const libnutModule = await import("@computer-use/libnut/dist/import_libnut");
+        device_libnut = libnutModule.libnut;
+        if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
+        return device_libnut;
+    } catch (error) {
+        libnutLoadError = error;
+        throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
+    }
+}
+const debugDevice = getDebug('computer:device');
+async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
+    node_assert(device_libnut, 'libnut not initialized');
+    const currentPos = device_libnut.getMousePos();
+    for(let i = 1; i <= steps; i++){
+        const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
+        const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
+        device_libnut.moveMouse(stepX, stepY);
+        await sleep(stepDelay);
+    }
+}
+const KEY_NAME_MAP = {
+    windows: 'win',
+    win: 'win',
+    ctrl: 'control',
+    esc: 'escape',
+    del: 'delete',
+    ins: 'insert',
+    pgup: 'pageup',
+    pgdn: 'pagedown',
+    arrowup: 'up',
+    arrowdown: 'down',
+    arrowleft: 'left',
+    arrowright: 'right',
+    volumedown: 'audio_vol_down',
+    volumeup: 'audio_vol_up',
+    mediavolumedown: 'audio_vol_down',
+    mediavolumeup: 'audio_vol_up',
+    mute: 'audio_mute',
+    mediamute: 'audio_mute',
+    mediaplay: 'audio_play',
+    mediapause: 'audio_pause',
+    mediaplaypause: 'audio_play',
+    mediastop: 'audio_stop',
+    medianexttrack: 'audio_next',
+    mediaprevioustrack: 'audio_prev',
+    medianext: 'audio_next',
+    mediaprev: 'audio_prev'
+};
+const PRIMARY_KEY_MAP = {
+    command: 'cmd',
+    cmd: 'cmd',
+    meta: 'meta',
+    control: 'control',
+    ctrl: 'control',
+    shift: 'shift',
+    alt: 'alt',
+    option: 'alt'
+};
+function normalizeKeyName(key) {
+    const lowerKey = key.toLowerCase();
+    return KEY_NAME_MAP[lowerKey] || lowerKey;
+}
+function normalizePrimaryKey(key) {
+    const lowerKey = key.toLowerCase();
+    if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
+    return KEY_NAME_MAP[lowerKey] || lowerKey;
+}
+class ComputerDevice {
+    describe() {
+        return this.description || 'Computer Device';
+    }
+    static async listDisplays() {
+        try {
+            const displays = await screenshot_desktop.listDisplays();
+            return displays.map((d)=>({
+                    id: String(d.id),
+                    name: d.name || `Display ${d.id}`,
+                    primary: d.primary || false
+                }));
+        } catch (error) {
+            debugDevice(`Failed to list displays: ${error}`);
+            return [];
+        }
+    }
+    async connect() {
+        debugDevice('Connecting to computer device');
+        try {
+            device_libnut = await getLibnut();
+            const size = await this.size();
+            const displays = await ComputerDevice.listDisplays();
+            this.description = `
+Type: Computer
+Platform: ${process.platform}
+Display: ${this.displayId || 'Primary'}
+Screen Size: ${size.width}x${size.height}
+Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}
+`;
+            debugDevice('Computer device connected', this.description);
+        } catch (error) {
+            debugDevice(`Failed to connect: ${error}`);
+            throw new Error(`Unable to connect to computer device: ${error}`);
+        }
+    }
+    async screenshotBase64() {
+        debugDevice('Taking screenshot', {
+            displayId: this.displayId
+        });
+        try {
+            const options = {
+                format: 'png'
+            };
+            if (void 0 !== this.displayId) if ('darwin' === process.platform) {
+                const screenIndex = Number(this.displayId);
+                if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
+            } else options.screen = this.displayId;
+            debugDevice('Screenshot options', options);
+            const buffer = await screenshot_desktop(options);
+            return createImgBase64ByFormat('png', buffer.toString('base64'));
+        } catch (error) {
+            debugDevice(`Screenshot failed: ${error}`);
+            throw new Error(`Failed to take screenshot: ${error}`);
+        }
+    }
+    async size() {
+        node_assert(device_libnut, 'libnut not initialized');
+        try {
+            const screenSize = device_libnut.getScreenSize();
+            return {
+                width: screenSize.width,
+                height: screenSize.height,
+                dpr: 1
+            };
+        } catch (error) {
+            debugDevice(`Failed to get screen size: ${error}`);
+            throw new Error(`Failed to get screen size: ${error}`);
+        }
+    }
+    actionSpace() {
+        const defaultActions = [
+            defineActionTap(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                const element = param.locate;
+                node_assert(element, 'Element not found, cannot tap');
+                const [x, y] = element.center;
+                const targetX = Math.round(x);
+                const targetY = Math.round(y);
+                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
+                device_libnut.mouseToggle('down', 'left');
+                await sleep(CLICK_HOLD_DURATION);
+                device_libnut.mouseToggle('up', 'left');
+            }),
+            defineActionDoubleClick(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                const element = param.locate;
+                node_assert(element, 'Element not found, cannot double click');
+                const [x, y] = element.center;
+                device_libnut.moveMouse(Math.round(x), Math.round(y));
+                device_libnut.mouseClick('left', true);
+            }),
+            defineActionRightClick(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                const element = param.locate;
+                node_assert(element, 'Element not found, cannot right click');
+                const [x, y] = element.center;
+                device_libnut.moveMouse(Math.round(x), Math.round(y));
+                device_libnut.mouseClick('right');
+            }),
+            defineActionHover(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                const element = param.locate;
+                node_assert(element, 'Element not found, cannot hover');
+                const [x, y] = element.center;
+                const targetX = Math.round(x);
+                const targetY = Math.round(y);
+                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_HOVER, SMOOTH_MOVE_DELAY_HOVER);
+                await sleep(HOVER_EFFECT_WAIT);
+            }),
+            defineAction({
+                name: 'Input',
+                description: 'Input text into the input field',
+                interfaceAlias: 'aiInput',
+                paramSchema: z.object({
+                    value: z.string().describe('The text to input'),
+                    mode: z["enum"]([
+                        'replace',
+                        'clear',
+                        'append'
+                    ]).default('replace').optional().describe('Input mode: replace, clear, or append'),
+                    locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
+                }),
+                call: async (param)=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    const element = param.locate;
+                    if (element && 'append' !== param.mode) {
+                        const [x, y] = element.center;
+                        device_libnut.moveMouse(Math.round(x), Math.round(y));
+                        device_libnut.mouseClick('left');
+                        await sleep(INPUT_FOCUS_DELAY);
+                        const modifier = 'darwin' === process.platform ? 'command' : 'control';
+                        device_libnut.keyTap('a', [
+                            modifier
+                        ]);
+                        await sleep(50);
+                        device_libnut.keyTap('backspace');
+                        await sleep(INPUT_CLEAR_DELAY);
+                    }
+                    if ('clear' === param.mode) return;
+                    if (!param.value) return;
+                    device_libnut.typeString(param.value);
+                }
+            }),
+            defineActionScroll(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                if (param.locate) {
+                    const element = param.locate;
+                    const [x, y] = element.center;
+                    device_libnut.moveMouse(Math.round(x), Math.round(y));
+                }
+                const scrollType = param?.scrollType;
+                const scrollToEdgeActions = {
+                    scrollToTop: [
+                        0,
+                        10
+                    ],
+                    scrollToBottom: [
+                        0,
+                        -10
+                    ],
+                    scrollToLeft: [
+                        -10,
+                        0
+                    ],
+                    scrollToRight: [
+                        10,
+                        0
+                    ]
+                };
+                const edgeAction = scrollToEdgeActions[scrollType || ''];
+                if (edgeAction) {
+                    const [dx, dy] = edgeAction;
+                    for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
+                        device_libnut.scrollMouse(dx, dy);
+                        await sleep(SCROLL_STEP_DELAY);
+                    }
+                    return;
+                }
+                if ('singleAction' === scrollType || !scrollType) {
+                    const distance = param?.distance || 500;
+                    const ticks = Math.ceil(distance / 100);
+                    const direction = param?.direction || 'down';
+                    const directionMap = {
+                        up: [
+                            0,
+                            ticks
+                        ],
+                        down: [
+                            0,
+                            -ticks
+                        ],
+                        left: [
+                            -ticks,
+                            0
+                        ],
+                        right: [
+                            ticks,
+                            0
+                        ]
+                    };
+                    const [dx, dy] = directionMap[direction] || [
+                        0,
+                        -ticks
+                    ];
+                    device_libnut.scrollMouse(dx, dy);
+                    await sleep(SCROLL_COMPLETE_DELAY);
+                    return;
+                }
+                throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
+            }),
+            defineActionKeyboardPress(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                if (param.locate) {
+                    const [x, y] = param.locate.center;
+                    device_libnut.moveMouse(Math.round(x), Math.round(y));
+                    device_libnut.mouseClick('left');
+                    await sleep(50);
+                }
+                const keys = param.keyName.split('+');
+                const modifiers = keys.slice(0, -1).map(normalizeKeyName);
+                const key = normalizePrimaryKey(keys[keys.length - 1]);
+                debugDevice('KeyboardPress', {
+                    original: param.keyName,
+                    key,
+                    modifiers
+                });
+                if (modifiers.length > 0) device_libnut.keyTap(key, modifiers);
+                else device_libnut.keyTap(key);
+            }),
+            defineActionDragAndDrop(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                const from = param.from;
+                const to = param.to;
+                node_assert(from, 'missing "from" param for drag and drop');
+                node_assert(to, 'missing "to" param for drag and drop');
+                const [fromX, fromY] = from.center;
+                const [toX, toY] = to.center;
+                device_libnut.moveMouse(Math.round(fromX), Math.round(fromY));
+                device_libnut.mouseToggle('down', 'left');
+                await sleep(100);
+                device_libnut.moveMouse(Math.round(toX), Math.round(toY));
+                await sleep(100);
+                device_libnut.mouseToggle('up', 'left');
+            }),
+            defineActionClearInput(async (param)=>{
+                node_assert(device_libnut, 'libnut not initialized');
+                const element = param.locate;
+                node_assert(element, 'Element not found, cannot clear input');
+                const [x, y] = element.center;
+                device_libnut.moveMouse(Math.round(x), Math.round(y));
+                device_libnut.mouseClick('left');
+                await sleep(100);
+                const modifier = 'darwin' === process.platform ? 'command' : 'control';
+                device_libnut.keyTap('a', [
+                    modifier
+                ]);
+                device_libnut.keyTap('backspace');
+                await sleep(50);
+            })
+        ];
+        const platformActions = Object.values(createPlatformActions());
+        const customActions = this.options?.customActions || [];
+        return [
+            ...defaultActions,
+            ...platformActions,
+            ...customActions
+        ];
+    }
+    async destroy() {
+        if (this.destroyed) return;
+        this.destroyed = true;
+        debugDevice('Computer device destroyed');
+    }
+    async url() {
+        return '';
+    }
+    constructor(options){
+        _define_property(this, "interfaceType", 'computer');
+        _define_property(this, "options", void 0);
+        _define_property(this, "displayId", void 0);
+        _define_property(this, "description", void 0);
+        _define_property(this, "destroyed", false);
+        _define_property(this, "uri", void 0);
+        this.options = options;
+        this.displayId = options?.displayId;
+    }
+}
+function createPlatformActions() {
+    return {
+        ListDisplays: defineAction({
+            name: 'ListDisplays',
+            description: 'List all available displays/monitors',
+            call: async ()=>await ComputerDevice.listDisplays()
+        })
+    };
+}
+class ComputerAgent extends Agent {
+}
+async function agentFromComputer(opts) {
+    const device = new ComputerDevice(opts || {});
+    await device.connect();
+    return new ComputerAgent(device, opts);
+}
+async function checkComputerEnvironment() {
+    try {
+        const libnutModule = await import("@computer-use/libnut/dist/import_libnut");
+        const libnut = libnutModule.libnut;
+        const screenSize = libnut.getScreenSize();
+        if (!screenSize || screenSize.width <= 0) return {
+            available: false,
+            error: 'libnut cannot get screen size',
+            platform: process.platform,
+            displays: 0
+        };
+        const displays = await ComputerDevice.listDisplays();
+        return {
+            available: true,
+            platform: process.platform,
+            displays: displays.length
+        };
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return {
+            available: false,
+            error: errorMessage,
+            platform: process.platform,
+            displays: 0
+        };
+    }
+}
+async function getConnectedDisplays() {
+    return ComputerDevice.listDisplays();
+}
+export { ComputerAgent, ComputerDevice, agentFromComputer, checkComputerEnvironment, getConnectedDisplays, overrideAIConfig };