@midscene/computer 1.8.0 → 1.8.1

package/dist/es/index.mjs

@@ -4,8 +4,7 @@ import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, resolve as external_node_path_resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { getMidsceneLocationSchema, z } from "@midscene/core";
-import { actionHoverParamSchema, actionTapParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
+import { defineAction, defineActionsFromInputPrimitives } from "@midscene/core/device";
 import { sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
@@ -13,6 +12,7 @@ import screenshot_desktop from "screenshot-desktop";
 import { Agent } from "@midscene/core/agent";
 import { once } from "node:events";
 import { createInterface } from "node:readline";
+import { z } from "@midscene/core";
 import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
 import { overrideAIConfig } from "@midscene/shared/env";
 const debugXvfb = getDebug('computer:xvfb');
@@ -92,15 +92,6 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const computerInputParamSchema = z.object({
-    value: z.string().describe('The text to input'),
-    mode: z["enum"]([
-        'replace',
-        'clear',
-        'append'
-    ]).default('replace').optional().describe('Input mode: replace, clear, or append'),
-    locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
-});
 const SMOOTH_MOVE_STEPS_TAP = 8;
 const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
 const SMOOTH_MOVE_DELAY_TAP = 8;
@@ -411,7 +402,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck() {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.8.0");
+        console.log("[HealthCheck] @midscene/computer v1.8.1");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -477,21 +468,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         debugDevice('Taking screenshot', {
             displayId: this.displayId
         });
-        try {
-            const options = {
-                format: 'png'
-            };
-            if (void 0 !== this.displayId) if ('darwin' === process.platform) {
-                const screenIndex = Number(this.displayId);
-                if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
-            } else options.screen = this.displayId;
-            debugDevice('Screenshot options', options);
+        const options = {
+            format: 'png'
+        };
+        if (void 0 !== this.displayId) if ('darwin' === process.platform) {
+            const screenIndex = Number(this.displayId);
+            if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
+        } else options.screen = this.displayId;
+        debugDevice('Screenshot options', options);
+        const MAX_ATTEMPTS = 3;
+        const RETRY_DELAY_MS = 300;
+        let lastRawMessage = '';
+        for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
             const buffer = await screenshot_desktop(options);
+            if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
             return createImgBase64ByFormat('png', buffer.toString('base64'));
         } catch (error) {
-            debugDevice(`Screenshot failed: ${error}`);
-            throw new Error(`Failed to take screenshot: ${error}`);
+            lastRawMessage = error instanceof Error ? error.message : String(error);
+            const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
+            const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
+            debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
+            if (!willRetry) break;
+            await sleep(RETRY_DELAY_MS);
         }
+        if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
+
+Please follow these steps:
+1. Open System Settings > Privacy & Security > Screen Recording
+2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
+3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
+
+Original error: ${lastRawMessage}`);
+        throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
     }
     async size() {
         node_assert(device_libnut, 'libnut not initialized');
@@ -537,228 +545,111 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         node_assert(device_libnut, 'libnut not initialized');
         await this.typeViaClipboard(text);
     }
+    async selectAllAndDelete() {
+        node_assert(device_libnut, 'libnut not initialized');
+        if (this.useAppleScript) {
+            sendKeyViaAppleScript('a', [
+                'command'
+            ]);
+            await sleep(50);
+            sendKeyViaAppleScript('backspace', []);
+            return;
+        }
+        const modifier = 'darwin' === process.platform ? 'command' : 'control';
+        device_libnut.keyTap('a', [
+            modifier
+        ]);
+        await sleep(50);
+        device_libnut.keyTap('backspace');
+    }
+    async pressKeyboardShortcut(keyName) {
+        node_assert(device_libnut, 'libnut not initialized');
+        const keys = keyName.split('+');
+        const modifiers = keys.slice(0, -1).map(normalizeKeyName);
+        const key = normalizePrimaryKey(keys[keys.length - 1]);
+        debugDevice('KeyboardPress', {
+            original: keyName,
+            key,
+            modifiers,
+            driver: this.useAppleScript ? "applescript" : 'libnut'
+        });
+        if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
+        else if (modifiers.length > 0) device_libnut.keyTap(key, modifiers);
+        else device_libnut.keyTap(key);
+    }
+    async performScroll(param) {
+        node_assert(device_libnut, 'libnut not initialized');
+        if (param.locate) {
+            const element = param.locate;
+            const [x, y] = element.center;
+            device_libnut.moveMouse(Math.round(x), Math.round(y));
+        }
+        const scrollType = param?.scrollType;
+        const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
+        if (edgeSpec) {
+            if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
+            if (this.useAppleScript) {
+                sendKeyViaAppleScript(edgeSpec.key);
+                await sleep(SCROLL_COMPLETE_DELAY);
+                return;
+            }
+            const [dx, dy] = edgeSpec.libnut;
+            for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
+                device_libnut.scrollMouse(dx, dy);
+                await sleep(SCROLL_STEP_DELAY);
+            }
+            return;
+        }
+        if ('singleAction' === scrollType || !scrollType) {
+            const distance = param?.distance || 500;
+            const direction = param?.direction || 'down';
+            const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
+            if (isKnownDirection) {
+                const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
+                if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
+            }
+            if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
+                const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
+                const key = 'up' === direction ? 'pageup' : 'pagedown';
+                for(let i = 0; i < pages; i++){
+                    sendKeyViaAppleScript(key);
+                    await sleep(SCROLL_STEP_DELAY);
+                }
+                await sleep(SCROLL_COMPLETE_DELAY);
+                return;
+            }
+            const ticks = Math.ceil(distance / 100);
+            const directionMap = {
+                up: [
+                    0,
+                    ticks
+                ],
+                down: [
+                    0,
+                    -ticks
+                ],
+                left: [
+                    -ticks,
+                    0
+                ],
+                right: [
+                    ticks,
+                    0
+                ]
+            };
+            const [dx, dy] = directionMap[direction] || [
+                0,
+                -ticks
+            ];
+            device_libnut.scrollMouse(dx, dy);
+            await sleep(SCROLL_COMPLETE_DELAY);
+            return;
+        }
+        throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
+    }
     actionSpace() {
         const defaultActions = [
-            defineActionTap(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot tap');
-                const [x, y] = element.center;
-                const targetX = Math.round(x);
-                const targetY = Math.round(y);
-                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
-                device_libnut.mouseToggle('down', 'left');
-                await sleep(CLICK_HOLD_DURATION);
-                device_libnut.mouseToggle('up', 'left');
-            }),
-            defineActionDoubleClick(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot double click');
-                const [x, y] = element.center;
-                device_libnut.moveMouse(Math.round(x), Math.round(y));
-                device_libnut.mouseClick('left', true);
-            }),
-            defineActionRightClick(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot right click');
-                const [x, y] = element.center;
-                device_libnut.moveMouse(Math.round(x), Math.round(y));
-                device_libnut.mouseClick('right');
-            }),
-            defineAction({
-                name: 'MouseMove',
-                description: 'Move the mouse to the element',
-                interfaceAlias: 'aiHover',
-                paramSchema: actionHoverParamSchema,
-                sample: {
-                    locate: {
-                        prompt: 'the navigation menu item "Products"'
-                    }
-                },
-                call: async (param)=>{
-                    node_assert(device_libnut, 'libnut not initialized');
-                    const element = param.locate;
-                    node_assert(element, 'Element not found, cannot move mouse');
-                    const [x, y] = element.center;
-                    const targetX = Math.round(x);
-                    const targetY = Math.round(y);
-                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
-                    await sleep(MOUSE_MOVE_EFFECT_WAIT);
-                }
-            }),
-            defineAction({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: computerInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    node_assert(device_libnut, 'libnut not initialized');
-                    const element = param.locate;
-                    if (element) {
-                        const [x, y] = element.center;
-                        device_libnut.moveMouse(Math.round(x), Math.round(y));
-                        device_libnut.mouseClick('left');
-                        await sleep(INPUT_FOCUS_DELAY);
-                        if ('append' !== param.mode) {
-                            if (this.useAppleScript) {
-                                sendKeyViaAppleScript('a', [
-                                    'command'
-                                ]);
-                                await sleep(50);
-                                sendKeyViaAppleScript('backspace', []);
-                            } else {
-                                const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                                device_libnut.keyTap('a', [
-                                    modifier
-                                ]);
-                                await sleep(50);
-                                device_libnut.keyTap('backspace');
-                            }
-                            await sleep(INPUT_CLEAR_DELAY);
-                        }
-                    }
-                    if ('clear' === param.mode) return;
-                    if (!param.value) return;
-                    await this.smartTypeString(param.value);
-                }
-            }),
-            defineActionScroll(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                if (param.locate) {
-                    const element = param.locate;
-                    const [x, y] = element.center;
-                    device_libnut.moveMouse(Math.round(x), Math.round(y));
-                }
-                const scrollType = param?.scrollType;
-                const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
-                if (edgeSpec) {
-                    if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
-                    if (this.useAppleScript) {
-                        sendKeyViaAppleScript(edgeSpec.key);
-                        await sleep(SCROLL_COMPLETE_DELAY);
-                        return;
-                    }
-                    const [dx, dy] = edgeSpec.libnut;
-                    for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
-                        device_libnut.scrollMouse(dx, dy);
-                        await sleep(SCROLL_STEP_DELAY);
-                    }
-                    return;
-                }
-                if ('singleAction' === scrollType || !scrollType) {
-                    const distance = param?.distance || 500;
-                    const direction = param?.direction || 'down';
-                    const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
-                    if (isKnownDirection) {
-                        const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
-                        if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
-                    }
-                    if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
-                        const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
-                        const key = 'up' === direction ? 'pageup' : 'pagedown';
-                        for(let i = 0; i < pages; i++){
-                            sendKeyViaAppleScript(key);
-                            await sleep(SCROLL_STEP_DELAY);
-                        }
-                        await sleep(SCROLL_COMPLETE_DELAY);
-                        return;
-                    }
-                    const ticks = Math.ceil(distance / 100);
-                    const directionMap = {
-                        up: [
-                            0,
-                            ticks
-                        ],
-                        down: [
-                            0,
-                            -ticks
-                        ],
-                        left: [
-                            -ticks,
-                            0
-                        ],
-                        right: [
-                            ticks,
-                            0
-                        ]
-                    };
-                    const [dx, dy] = directionMap[direction] || [
-                        0,
-                        -ticks
-                    ];
-                    device_libnut.scrollMouse(dx, dy);
-                    await sleep(SCROLL_COMPLETE_DELAY);
-                    return;
-                }
-                throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
-            }),
-            defineActionKeyboardPress(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                if (param.locate) {
-                    const [x, y] = param.locate.center;
-                    device_libnut.moveMouse(Math.round(x), Math.round(y));
-                    device_libnut.mouseClick('left');
-                    await sleep(50);
-                }
-                const keys = param.keyName.split('+');
-                const modifiers = keys.slice(0, -1).map(normalizeKeyName);
-                const key = normalizePrimaryKey(keys[keys.length - 1]);
-                debugDevice('KeyboardPress', {
-                    original: param.keyName,
-                    key,
-                    modifiers,
-                    driver: this.useAppleScript ? "applescript" : 'libnut'
-                });
-                if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
-                else if (modifiers.length > 0) device_libnut.keyTap(key, modifiers);
-                else device_libnut.keyTap(key);
-            }),
-            defineActionDragAndDrop(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                const from = param.from;
-                const to = param.to;
-                node_assert(from, 'missing "from" param for drag and drop');
-                node_assert(to, 'missing "to" param for drag and drop');
-                const [fromX, fromY] = from.center;
-                const [toX, toY] = to.center;
-                device_libnut.moveMouse(Math.round(fromX), Math.round(fromY));
-                device_libnut.mouseToggle('down', 'left');
-                await sleep(100);
-                device_libnut.moveMouse(Math.round(toX), Math.round(toY));
-                await sleep(100);
-                device_libnut.mouseToggle('up', 'left');
-            }),
-            defineActionClearInput(async (param)=>{
-                node_assert(device_libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot clear input');
-                const [x, y] = element.center;
-                device_libnut.moveMouse(Math.round(x), Math.round(y));
-                device_libnut.mouseClick('left');
-                await sleep(100);
-                if (this.useAppleScript) {
-                    sendKeyViaAppleScript('a', [
-                        'command'
-                    ]);
-                    await sleep(50);
-                    sendKeyViaAppleScript('backspace', []);
-                } else {
-                    const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                    device_libnut.keyTap('a', [
-                        modifier
-                    ]);
-                    device_libnut.keyTap('backspace');
-                }
-                await sleep(50);
-            })
+            ...defineActionsFromInputPrimitives(this.inputPrimitives)
         ];
         const platformActions = Object.values(createPlatformActions());
         const customActions = this.options?.customActions || [];
@@ -796,6 +687,88 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         _define_property(this, "xvfbCleanup", void 0);
         _define_property(this, "useAppleScript", void 0);
         _define_property(this, "uri", void 0);
+        _define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: async ({ x, y })=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    const targetX = Math.round(x);
+                    const targetY = Math.round(y);
+                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
+                    device_libnut.mouseToggle('down', 'left');
+                    await sleep(CLICK_HOLD_DURATION);
+                    device_libnut.mouseToggle('up', 'left');
+                },
+                doubleClick: async ({ x, y })=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    device_libnut.moveMouse(Math.round(x), Math.round(y));
+                    device_libnut.mouseClick('left', true);
+                },
+                rightClick: async ({ x, y })=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    device_libnut.moveMouse(Math.round(x), Math.round(y));
+                    device_libnut.mouseClick('right');
+                },
+                hover: async ({ x, y })=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    await smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
+                    await sleep(MOUSE_MOVE_EFFECT_WAIT);
+                },
+                dragAndDrop: async (from, to)=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    device_libnut.moveMouse(Math.round(from.x), Math.round(from.y));
+                    device_libnut.mouseToggle('down', 'left');
+                    await sleep(100);
+                    device_libnut.moveMouse(Math.round(to.x), Math.round(to.y));
+                    await sleep(100);
+                    device_libnut.mouseToggle('up', 'left');
+                }
+            },
+            keyboard: {
+                typeText: async (value, opts)=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    const element = opts?.target;
+                    if (element) {
+                        const [x, y] = element.center;
+                        device_libnut.moveMouse(Math.round(x), Math.round(y));
+                        device_libnut.mouseClick('left');
+                        await sleep(INPUT_FOCUS_DELAY);
+                        if (opts?.replace !== false) {
+                            await this.selectAllAndDelete();
+                            await sleep(INPUT_CLEAR_DELAY);
+                        }
+                    }
+                    await this.smartTypeString(value);
+                },
+                keyboardPress: async (keyName, opts)=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    const target = opts?.target;
+                    if (target) {
+                        const [x, y] = target.center;
+                        device_libnut.moveMouse(Math.round(x), Math.round(y));
+                        device_libnut.mouseClick('left');
+                        await sleep(50);
+                    }
+                    await this.pressKeyboardShortcut(keyName);
+                },
+                clearInput: async (target)=>{
+                    node_assert(device_libnut, 'libnut not initialized');
+                    if (target) {
+                        const element = target;
+                        const [x, y] = element.center;
+                        device_libnut.moveMouse(Math.round(x), Math.round(y));
+                        device_libnut.mouseClick('left');
+                        await sleep(100);
+                    }
+                    await this.selectAllAndDelete();
+                    await sleep(50);
+                }
+            },
+            scroll: {
+                scroll: async (param)=>{
+                    await this.performScroll(param);
+                }
+            }
+        });
         this.options = options;
         this.displayId = options?.displayId;
         this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -1214,132 +1187,7 @@ class RDPDevice {
     }
     actionSpace() {
         const defaultActions = [
-            defineActionTap(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'tap');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'down');
-                await sleep(device_CLICK_HOLD_DURATION);
-                await this.backend.mouseButton('left', 'up');
-            }),
-            defineActionDoubleClick(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'double click');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'doubleClick');
-            }),
-            defineActionRightClick(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'right click');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('right', 'click');
-            }),
-            defineActionHover(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'hover');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
-                    settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
-                });
-            }),
-            defineActionInput(async (param)=>{
-                this.assertConnected();
-                if (param.locate) {
-                    await this.moveToElement(param.locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                    await sleep(device_INPUT_FOCUS_DELAY);
-                }
-                if ('typeOnly' !== param.mode) {
-                    await this.clearInput();
-                    await sleep(device_INPUT_CLEAR_DELAY);
-                }
-                if ('clear' === param.mode) return;
-                if (param.value) await this.backend.typeText(param.value);
-            }),
-            defineActionClearInput(async ({ locate })=>{
-                this.assertConnected();
-                if (locate) {
-                    await this.moveToElement(locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                    await sleep(device_INPUT_FOCUS_DELAY);
-                }
-                await this.clearInput();
-                await sleep(device_INPUT_CLEAR_DELAY);
-            }),
-            defineActionKeyboardPress(async ({ locate, keyName })=>{
-                this.assertConnected();
-                if (locate) {
-                    await this.moveToElement(locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                }
-                await this.backend.keyPress(keyName);
-            }),
-            defineActionScroll(async (param)=>{
-                this.assertConnected();
-                const target = param.locate;
-                if (target) await this.moveToElement(target, {
-                    steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
-                });
-                if (param.scrollType && 'singleAction' !== param.scrollType) {
-                    const direction = this.edgeScrollDirection(param.scrollType);
-                    for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
-                    await sleep(device_SCROLL_COMPLETE_DELAY);
-                    return;
-                }
-                await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
-                await sleep(device_SCROLL_COMPLETE_DELAY);
-            }),
-            defineActionDragAndDrop(async ({ from, to })=>{
-                this.assertConnected();
-                const source = this.requireLocate(from, 'drag source');
-                const target = this.requireLocate(to, 'drag target');
-                await this.moveToElement(source, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'down');
-                await sleep(DRAG_HOLD_DURATION);
-                await this.moveToElement(target, {
-                    steps: SMOOTH_MOVE_STEPS_DRAG,
-                    stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
-                });
-                await sleep(DRAG_HOLD_DURATION);
-                await this.backend.mouseButton('left', 'up');
-            }),
-            defineAction({
-                name: 'MiddleClick',
-                description: 'Middle click the element',
-                sample: {
-                    locate: {
-                        prompt: 'the browser tab close target'
-                    }
-                },
-                paramSchema: actionTapParamSchema,
-                call: async ({ locate })=>{
-                    const element = this.requireLocate(locate, 'middle click');
-                    await this.moveToElement(element, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('middle', 'click');
-                }
-            }),
+            ...defineActionsFromInputPrimitives(this.inputPrimitives),
             defineAction({
                 name: 'ListDisplays',
                 description: 'List all available displays/monitors',
@@ -1368,10 +1216,6 @@ class RDPDevice {
     throwIfDestroyed() {
         if (this.destroyed) throw new Error('RDPDevice has been destroyed');
     }
-    requireLocate(locate, actionName) {
-        if (!locate) throw new Error(`Missing target element for ${actionName}`);
-        return locate;
-    }
     async moveToElement(element, options) {
         this.assertConnected();
         const targetX = Math.round(element.center[0]);
@@ -1437,6 +1281,113 @@ class RDPDevice {
         device_define_property(this, "destroyed", false);
         device_define_property(this, "cursorPosition", void 0);
         device_define_property(this, "uri", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'down');
+                    await sleep(device_CLICK_HOLD_DURATION);
+                    await this.backend.mouseButton('left', 'up');
+                },
+                doubleClick: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'doubleClick');
+                },
+                rightClick: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('right', 'click');
+                },
+                hover: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
+                        settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
+                    });
+                },
+                dragAndDrop: async (from, to)=>{
+                    await this.movePointer(Math.round(from.x), Math.round(from.y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'down');
+                    await sleep(DRAG_HOLD_DURATION);
+                    await this.movePointer(Math.round(to.x), Math.round(to.y), {
+                        steps: SMOOTH_MOVE_STEPS_DRAG,
+                        stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
+                    });
+                    await sleep(DRAG_HOLD_DURATION);
+                    await this.backend.mouseButton('left', 'up');
+                }
+            },
+            keyboard: {
+                typeText: async (value, opts)=>{
+                    this.assertConnected();
+                    const target = opts?.target;
+                    if (target) {
+                        await this.inputPrimitives.pointer.tap({
+                            x: target.center[0],
+                            y: target.center[1]
+                        });
+                        await sleep(device_INPUT_FOCUS_DELAY);
+                    }
+                    if (opts?.replace !== false) {
+                        await this.clearInput();
+                        await sleep(device_INPUT_CLEAR_DELAY);
+                    }
+                    if (opts?.focusOnly || !value) return;
+                    await this.backend.typeText(value);
+                },
+                clearInput: async (target)=>{
+                    this.assertConnected();
+                    const element = target;
+                    if (element) {
+                        await this.inputPrimitives.pointer.tap({
+                            x: element.center[0],
+                            y: element.center[1]
+                        });
+                        await sleep(device_INPUT_FOCUS_DELAY);
+                    }
+                    await this.clearInput();
+                    await sleep(device_INPUT_CLEAR_DELAY);
+                },
+                keyboardPress: async (keyName, opts)=>{
+                    this.assertConnected();
+                    const target = opts?.target;
+                    if (target) await this.inputPrimitives.pointer.tap({
+                        x: target.center[0],
+                        y: target.center[1]
+                    });
+                    await this.backend.keyPress(keyName);
+                }
+            },
+            scroll: {
+                scroll: async (param)=>{
+                    this.assertConnected();
+                    const target = param.locate;
+                    if (target) await this.moveToElement(target, {
+                        steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
+                    });
+                    if (param.scrollType && 'singleAction' !== param.scrollType) {
+                        const direction = this.edgeScrollDirection(param.scrollType);
+                        for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
+                        await sleep(device_SCROLL_COMPLETE_DELAY);
+                        return;
+                    }
+                    await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
+                    await sleep(device_SCROLL_COMPLETE_DELAY);
+                }
+            }
+        });
         this.options = {
             port: 3389,
             securityProtocol: 'auto',
@@ -1495,10 +1446,61 @@ function mcp_tools_define_property(obj, key, value) {
     return obj;
 }
 const mcp_tools_debug = getDebug('mcp:computer-tools');
+const RDP_SECURITY_PROTOCOLS = [
+    'auto',
+    'tls',
+    'nla',
+    'rdp'
+];
 const computerInitArgShape = {
-    displayId: z.string().optional().describe('Display ID (from computer_list_displays)'),
-    headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
+    displayId: z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
+    headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
+    host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
+    port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
+    username: z.string().optional().describe('RDP username. Requires host.'),
+    password: z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
+    domain: z.string().optional().describe('RDP domain. Requires host.'),
+    adminSession: z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
+    ignoreCertificate: z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
+    securityProtocol: z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
+    desktopWidth: z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
+    desktopHeight: z.number().optional().describe('Remote desktop height in pixels. Requires host.')
 };
+function adaptComputerInitArgs(extracted) {
+    if (!extracted || 0 === Object.keys(extracted).length) return;
+    if (extracted.host) {
+        const { displayId: _d, headless: _h, ...rdpFields } = extracted;
+        return {
+            mode: 'rdp',
+            ...rdpFields,
+            host: extracted.host
+        };
+    }
+    return {
+        mode: 'local',
+        displayId: extracted.displayId,
+        headless: extracted.headless
+    };
+}
+function shouldRetargetAgent(opts) {
+    if (!opts) return false;
+    if ('rdp' === opts.mode) return true;
+    return void 0 !== opts.displayId || void 0 !== opts.headless;
+}
+function describeConnectTarget(opts) {
+    if (opts?.mode === 'rdp') {
+        const portSuffix = opts.port ? `:${opts.port}` : '';
+        const userSuffix = opts.username ? ` as ${opts.username}` : '';
+        return ` via RDP (${opts.host}${portSuffix}${userSuffix})`;
+    }
+    if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
+    return ' (Primary display)';
+}
+function getCliReportSessionTarget(opts) {
+    if (opts?.mode === 'rdp') return `rdp:${opts.host}`;
+    if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
+    return 'primary';
+}
 class ComputerMidsceneTools extends BaseMidsceneTools {
     getCliReportSessionName() {
         return 'midscene-computer';
@@ -1507,9 +1509,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
         return new ComputerDevice({});
     }
     async ensureAgent(opts) {
-        const displayId = opts?.displayId;
-        const headless = opts?.headless;
-        if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
+        if (this.agent && shouldRetargetAgent(opts)) {
             try {
                 await this.agent.destroy?.();
             } catch (error) {
@@ -1518,8 +1518,20 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
             this.agent = void 0;
         }
         if (this.agent) return this.agent;
-        mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
         const reportOptions = this.readCliReportAgentOptions();
+        if (opts?.mode === 'rdp') {
+            mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
+            const { mode: _mode, ...rdpFields } = opts;
+            const agent = await agentForRDPComputer({
+                ...rdpFields,
+                ...reportOptions ?? {}
+            });
+            this.agent = agent;
+            return agent;
+        }
+        const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
+        const headless = opts?.mode === 'local' ? opts.headless : void 0;
+        mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
         const agentOpts = {
             ...displayId ? {
                 displayId
@@ -1537,12 +1549,12 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
         return [
             {
                 name: 'computer_connect',
-                description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
+                description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
                 schema: this.getAgentInitArgSchema(),
                 cli: this.getAgentInitArgCliMetadata(),
                 handler: async (args)=>{
                     const initArgs = this.extractAgentInitParam(args);
-                    const reportSession = this.createNewCliReportSession(initArgs?.displayId ?? 'primary');
+                    const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
                     this.commitCliReportSession(reportSession);
                     if (this.agent) {
                         try {
@@ -1558,7 +1570,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
                         content: [
                             {
                                 type: 'text',
-                                text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
+                                text: `Connected to computer${describeConnectTarget(initArgs)}`
                             },
                             ...this.buildScreenshotContent(screenshot)
                         ]
@@ -1596,31 +1608,43 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
             cli: {
                 preferBareKeys: true
             },
-            adapt: (extracted)=>extracted
+            adapt: (extracted)=>adaptComputerInitArgs(extracted)
         });
     }
 }
 function version() {
-    const currentVersion = "1.8.0";
+    const currentVersion = "1.8.1";
     console.log(`@midscene/computer v${currentVersion}`);
     return currentVersion;
 }
+function loadMacPermissions() {
+    if ('darwin' !== process.platform) return {
+        permissions: null
+    };
+    try {
+        const dynamicRequire = createRequire(import.meta.url);
+        return {
+            permissions: dynamicRequire('node-mac-permissions')
+        };
+    } catch (error) {
+        return {
+            permissions: null,
+            loadError: error instanceof Error ? error.message : String(error)
+        };
+    }
+}
 function checkAccessibilityPermission(promptIfNeeded = false) {
     if ('darwin' !== process.platform) return {
         hasPermission: true,
         platform: process.platform
     };
     try {
-        let permissions;
-        try {
-            const dynamicRequire = createRequire(import.meta.url);
-            permissions = dynamicRequire('node-mac-permissions');
-        } catch  {
-            return {
-                hasPermission: true,
-                platform: process.platform
-            };
-        }
+        const { permissions, loadError } = loadMacPermissions();
+        if (!permissions) return {
+            hasPermission: false,
+            platform: process.platform,
+            error: `Cannot verify macOS Accessibility permission: node-mac-permissions is unavailable${loadError ? ` (${loadError})` : ''}. The native module may need to be rebuilt for the current Node/Electron ABI.`
+        };
         const status = permissions.getAuthStatus('accessibility');
         if ('authorized' === status) return {
             hasPermission: true,
@@ -1640,6 +1664,37 @@ function checkAccessibilityPermission(promptIfNeeded = false) {
         };
     }
 }
+function checkScreenRecordingPermission(promptIfNeeded = false) {
+    if ('darwin' !== process.platform) return {
+        hasPermission: true,
+        platform: process.platform
+    };
+    try {
+        const { permissions, loadError } = loadMacPermissions();
+        if (!permissions) return {
+            hasPermission: false,
+            platform: process.platform,
+            error: `Cannot verify macOS Screen Recording permission: node-mac-permissions is unavailable${loadError ? ` (${loadError})` : ''}. The native module may need to be rebuilt for the current Node/Electron ABI.`
+        };
+        const status = permissions.getAuthStatus('screen');
+        if ('authorized' === status) return {
+            hasPermission: true,
+            platform: process.platform
+        };
+        if (promptIfNeeded) permissions.askForScreenCaptureAccess(true);
+        return {
+            hasPermission: false,
+            platform: process.platform,
+            error: `macOS Screen Recording permission is required (current status: ${status}).\n\nPlease follow these steps:\n1. Open System Settings > Privacy & Security > Screen Recording\n2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)\n3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.`
+        };
+    } catch (error) {
+        return {
+            hasPermission: false,
+            platform: process.platform,
+            error: `Failed to check screen recording permission: ${error instanceof Error ? error.message : String(error)}`
+        };
+    }
+}
 async function checkComputerEnvironment() {
     try {
         const libnutModule = await import("@computer-use/libnut/dist/import_libnut.js");
@@ -1670,4 +1725,4 @@ async function checkComputerEnvironment() {
 async function getConnectedDisplays() {
     return ComputerDevice.listDisplays();
 }
-export { ComputerAgent, ComputerDevice, ComputerMidsceneTools, HelperProcessRDPBackendClient, RDPDevice, UnsupportedRDPBackendClient, agentForComputer, agentForRDPComputer, agentFromComputer, checkAccessibilityPermission, checkComputerEnvironment, checkXvfbInstalled, createDefaultRDPBackendClient, getConnectedDisplays, needsXvfb, overrideAIConfig, version };
+export { ComputerAgent, ComputerDevice, ComputerMidsceneTools, HelperProcessRDPBackendClient, RDPDevice, UnsupportedRDPBackendClient, agentForComputer, agentForRDPComputer, agentFromComputer, checkAccessibilityPermission, checkComputerEnvironment, checkScreenRecordingPermission, checkXvfbInstalled, createDefaultRDPBackendClient, getConnectedDisplays, needsXvfb, overrideAIConfig, version };