@midscene/computer 1.8.1-beta-20260513084557.0 → 1.8.1

package/dist/es/mcp-server.mjs

@@ -6,14 +6,14 @@ import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, resolve as external_node_path_resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { getMidsceneLocationSchema, z } from "@midscene/core";
-import { actionHoverParamSchema, actionTapParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionRightClick, defineActionScroll, defineActionTap } from "@midscene/core/device";
+import { defineAction, defineActionsFromInputPrimitives } from "@midscene/core/device";
 import { sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import screenshot_desktop from "screenshot-desktop";
 import { once } from "node:events";
 import { createInterface } from "node:readline";
+import { z } from "@midscene/core";
 import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
 const debugXvfb = getDebug('computer:xvfb');
 function checkXvfbInstalled() {
@@ -92,15 +92,6 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const computerInputParamSchema = z.object({
-    value: z.string().describe('The text to input'),
-    mode: z["enum"]([
-        'replace',
-        'clear',
-        'append'
-    ]).default('replace').optional().describe('Input mode: replace, clear, or append'),
-    locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
-});
 const SMOOTH_MOVE_STEPS_TAP = 8;
 const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
 const SMOOTH_MOVE_DELAY_TAP = 8;
@@ -411,7 +402,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck() {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.8.1-beta-20260513084557.0");
+        console.log("[HealthCheck] @midscene/computer v1.8.1");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -477,21 +468,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         debugDevice('Taking screenshot', {
             displayId: this.displayId
         });
-        try {
-            const options = {
-                format: 'png'
-            };
-            if (void 0 !== this.displayId) if ('darwin' === process.platform) {
-                const screenIndex = Number(this.displayId);
-                if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
-            } else options.screen = this.displayId;
-            debugDevice('Screenshot options', options);
+        const options = {
+            format: 'png'
+        };
+        if (void 0 !== this.displayId) if ('darwin' === process.platform) {
+            const screenIndex = Number(this.displayId);
+            if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
+        } else options.screen = this.displayId;
+        debugDevice('Screenshot options', options);
+        const MAX_ATTEMPTS = 3;
+        const RETRY_DELAY_MS = 300;
+        let lastRawMessage = '';
+        for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
             const buffer = await screenshot_desktop(options);
+            if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
             return createImgBase64ByFormat('png', buffer.toString('base64'));
         } catch (error) {
-            debugDevice(`Screenshot failed: ${error}`);
-            throw new Error(`Failed to take screenshot: ${error}`);
+            lastRawMessage = error instanceof Error ? error.message : String(error);
+            const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
+            const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
+            debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
+            if (!willRetry) break;
+            await sleep(RETRY_DELAY_MS);
         }
+        if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
+
+Please follow these steps:
+1. Open System Settings > Privacy & Security > Screen Recording
+2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
+3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
+
+Original error: ${lastRawMessage}`);
+        throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
     }
     async size() {
         node_assert(libnut, 'libnut not initialized');
@@ -537,228 +545,111 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         node_assert(libnut, 'libnut not initialized');
         await this.typeViaClipboard(text);
     }
+    async selectAllAndDelete() {
+        node_assert(libnut, 'libnut not initialized');
+        if (this.useAppleScript) {
+            sendKeyViaAppleScript('a', [
+                'command'
+            ]);
+            await sleep(50);
+            sendKeyViaAppleScript('backspace', []);
+            return;
+        }
+        const modifier = 'darwin' === process.platform ? 'command' : 'control';
+        libnut.keyTap('a', [
+            modifier
+        ]);
+        await sleep(50);
+        libnut.keyTap('backspace');
+    }
+    async pressKeyboardShortcut(keyName) {
+        node_assert(libnut, 'libnut not initialized');
+        const keys = keyName.split('+');
+        const modifiers = keys.slice(0, -1).map(normalizeKeyName);
+        const key = normalizePrimaryKey(keys[keys.length - 1]);
+        debugDevice('KeyboardPress', {
+            original: keyName,
+            key,
+            modifiers,
+            driver: this.useAppleScript ? "applescript" : 'libnut'
+        });
+        if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
+        else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
+        else libnut.keyTap(key);
+    }
+    async performScroll(param) {
+        node_assert(libnut, 'libnut not initialized');
+        if (param.locate) {
+            const element = param.locate;
+            const [x, y] = element.center;
+            libnut.moveMouse(Math.round(x), Math.round(y));
+        }
+        const scrollType = param?.scrollType;
+        const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
+        if (edgeSpec) {
+            if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
+            if (this.useAppleScript) {
+                sendKeyViaAppleScript(edgeSpec.key);
+                await sleep(SCROLL_COMPLETE_DELAY);
+                return;
+            }
+            const [dx, dy] = edgeSpec.libnut;
+            for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
+                libnut.scrollMouse(dx, dy);
+                await sleep(SCROLL_STEP_DELAY);
+            }
+            return;
+        }
+        if ('singleAction' === scrollType || !scrollType) {
+            const distance = param?.distance || 500;
+            const direction = param?.direction || 'down';
+            const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
+            if (isKnownDirection) {
+                const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
+                if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
+            }
+            if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
+                const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
+                const key = 'up' === direction ? 'pageup' : 'pagedown';
+                for(let i = 0; i < pages; i++){
+                    sendKeyViaAppleScript(key);
+                    await sleep(SCROLL_STEP_DELAY);
+                }
+                await sleep(SCROLL_COMPLETE_DELAY);
+                return;
+            }
+            const ticks = Math.ceil(distance / 100);
+            const directionMap = {
+                up: [
+                    0,
+                    ticks
+                ],
+                down: [
+                    0,
+                    -ticks
+                ],
+                left: [
+                    -ticks,
+                    0
+                ],
+                right: [
+                    ticks,
+                    0
+                ]
+            };
+            const [dx, dy] = directionMap[direction] || [
+                0,
+                -ticks
+            ];
+            libnut.scrollMouse(dx, dy);
+            await sleep(SCROLL_COMPLETE_DELAY);
+            return;
+        }
+        throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
+    }
     actionSpace() {
         const defaultActions = [
-            defineActionTap(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot tap');
-                const [x, y] = element.center;
-                const targetX = Math.round(x);
-                const targetY = Math.round(y);
-                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
-                libnut.mouseToggle('down', 'left');
-                await sleep(CLICK_HOLD_DURATION);
-                libnut.mouseToggle('up', 'left');
-            }),
-            defineActionDoubleClick(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot double click');
-                const [x, y] = element.center;
-                libnut.moveMouse(Math.round(x), Math.round(y));
-                libnut.mouseClick('left', true);
-            }),
-            defineActionRightClick(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot right click');
-                const [x, y] = element.center;
-                libnut.moveMouse(Math.round(x), Math.round(y));
-                libnut.mouseClick('right');
-            }),
-            defineAction({
-                name: 'MouseMove',
-                description: 'Move the mouse to the element',
-                interfaceAlias: 'aiHover',
-                paramSchema: actionHoverParamSchema,
-                sample: {
-                    locate: {
-                        prompt: 'the navigation menu item "Products"'
-                    }
-                },
-                call: async (param)=>{
-                    node_assert(libnut, 'libnut not initialized');
-                    const element = param.locate;
-                    node_assert(element, 'Element not found, cannot move mouse');
-                    const [x, y] = element.center;
-                    const targetX = Math.round(x);
-                    const targetY = Math.round(y);
-                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
-                    await sleep(MOUSE_MOVE_EFFECT_WAIT);
-                }
-            }),
-            defineAction({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: computerInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    node_assert(libnut, 'libnut not initialized');
-                    const element = param.locate;
-                    if (element) {
-                        const [x, y] = element.center;
-                        libnut.moveMouse(Math.round(x), Math.round(y));
-                        libnut.mouseClick('left');
-                        await sleep(INPUT_FOCUS_DELAY);
-                        if ('append' !== param.mode) {
-                            if (this.useAppleScript) {
-                                sendKeyViaAppleScript('a', [
-                                    'command'
-                                ]);
-                                await sleep(50);
-                                sendKeyViaAppleScript('backspace', []);
-                            } else {
-                                const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                                libnut.keyTap('a', [
-                                    modifier
-                                ]);
-                                await sleep(50);
-                                libnut.keyTap('backspace');
-                            }
-                            await sleep(INPUT_CLEAR_DELAY);
-                        }
-                    }
-                    if ('clear' === param.mode) return;
-                    if (!param.value) return;
-                    await this.smartTypeString(param.value);
-                }
-            }),
-            defineActionScroll(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                if (param.locate) {
-                    const element = param.locate;
-                    const [x, y] = element.center;
-                    libnut.moveMouse(Math.round(x), Math.round(y));
-                }
-                const scrollType = param?.scrollType;
-                const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
-                if (edgeSpec) {
-                    if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
-                    if (this.useAppleScript) {
-                        sendKeyViaAppleScript(edgeSpec.key);
-                        await sleep(SCROLL_COMPLETE_DELAY);
-                        return;
-                    }
-                    const [dx, dy] = edgeSpec.libnut;
-                    for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
-                        libnut.scrollMouse(dx, dy);
-                        await sleep(SCROLL_STEP_DELAY);
-                    }
-                    return;
-                }
-                if ('singleAction' === scrollType || !scrollType) {
-                    const distance = param?.distance || 500;
-                    const direction = param?.direction || 'down';
-                    const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
-                    if (isKnownDirection) {
-                        const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
-                        if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
-                    }
-                    if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
-                        const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
-                        const key = 'up' === direction ? 'pageup' : 'pagedown';
-                        for(let i = 0; i < pages; i++){
-                            sendKeyViaAppleScript(key);
-                            await sleep(SCROLL_STEP_DELAY);
-                        }
-                        await sleep(SCROLL_COMPLETE_DELAY);
-                        return;
-                    }
-                    const ticks = Math.ceil(distance / 100);
-                    const directionMap = {
-                        up: [
-                            0,
-                            ticks
-                        ],
-                        down: [
-                            0,
-                            -ticks
-                        ],
-                        left: [
-                            -ticks,
-                            0
-                        ],
-                        right: [
-                            ticks,
-                            0
-                        ]
-                    };
-                    const [dx, dy] = directionMap[direction] || [
-                        0,
-                        -ticks
-                    ];
-                    libnut.scrollMouse(dx, dy);
-                    await sleep(SCROLL_COMPLETE_DELAY);
-                    return;
-                }
-                throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
-            }),
-            defineActionKeyboardPress(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                if (param.locate) {
-                    const [x, y] = param.locate.center;
-                    libnut.moveMouse(Math.round(x), Math.round(y));
-                    libnut.mouseClick('left');
-                    await sleep(50);
-                }
-                const keys = param.keyName.split('+');
-                const modifiers = keys.slice(0, -1).map(normalizeKeyName);
-                const key = normalizePrimaryKey(keys[keys.length - 1]);
-                debugDevice('KeyboardPress', {
-                    original: param.keyName,
-                    key,
-                    modifiers,
-                    driver: this.useAppleScript ? "applescript" : 'libnut'
-                });
-                if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
-                else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
-                else libnut.keyTap(key);
-            }),
-            defineActionDragAndDrop(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                const from = param.from;
-                const to = param.to;
-                node_assert(from, 'missing "from" param for drag and drop');
-                node_assert(to, 'missing "to" param for drag and drop');
-                const [fromX, fromY] = from.center;
-                const [toX, toY] = to.center;
-                libnut.moveMouse(Math.round(fromX), Math.round(fromY));
-                libnut.mouseToggle('down', 'left');
-                await sleep(100);
-                libnut.moveMouse(Math.round(toX), Math.round(toY));
-                await sleep(100);
-                libnut.mouseToggle('up', 'left');
-            }),
-            defineActionClearInput(async (param)=>{
-                node_assert(libnut, 'libnut not initialized');
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot clear input');
-                const [x, y] = element.center;
-                libnut.moveMouse(Math.round(x), Math.round(y));
-                libnut.mouseClick('left');
-                await sleep(100);
-                if (this.useAppleScript) {
-                    sendKeyViaAppleScript('a', [
-                        'command'
-                    ]);
-                    await sleep(50);
-                    sendKeyViaAppleScript('backspace', []);
-                } else {
-                    const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                    libnut.keyTap('a', [
-                        modifier
-                    ]);
-                    libnut.keyTap('backspace');
-                }
-                await sleep(50);
-            })
+            ...defineActionsFromInputPrimitives(this.inputPrimitives)
         ];
         const platformActions = Object.values(createPlatformActions());
         const customActions = this.options?.customActions || [];
@@ -796,6 +687,88 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         _define_property(this, "xvfbCleanup", void 0);
         _define_property(this, "useAppleScript", void 0);
         _define_property(this, "uri", void 0);
+        _define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: async ({ x, y })=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    const targetX = Math.round(x);
+                    const targetY = Math.round(y);
+                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
+                    libnut.mouseToggle('down', 'left');
+                    await sleep(CLICK_HOLD_DURATION);
+                    libnut.mouseToggle('up', 'left');
+                },
+                doubleClick: async ({ x, y })=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    libnut.moveMouse(Math.round(x), Math.round(y));
+                    libnut.mouseClick('left', true);
+                },
+                rightClick: async ({ x, y })=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    libnut.moveMouse(Math.round(x), Math.round(y));
+                    libnut.mouseClick('right');
+                },
+                hover: async ({ x, y })=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    await smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
+                    await sleep(MOUSE_MOVE_EFFECT_WAIT);
+                },
+                dragAndDrop: async (from, to)=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    libnut.moveMouse(Math.round(from.x), Math.round(from.y));
+                    libnut.mouseToggle('down', 'left');
+                    await sleep(100);
+                    libnut.moveMouse(Math.round(to.x), Math.round(to.y));
+                    await sleep(100);
+                    libnut.mouseToggle('up', 'left');
+                }
+            },
+            keyboard: {
+                typeText: async (value, opts)=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    const element = opts?.target;
+                    if (element) {
+                        const [x, y] = element.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await sleep(INPUT_FOCUS_DELAY);
+                        if (opts?.replace !== false) {
+                            await this.selectAllAndDelete();
+                            await sleep(INPUT_CLEAR_DELAY);
+                        }
+                    }
+                    await this.smartTypeString(value);
+                },
+                keyboardPress: async (keyName, opts)=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    const target = opts?.target;
+                    if (target) {
+                        const [x, y] = target.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await sleep(50);
+                    }
+                    await this.pressKeyboardShortcut(keyName);
+                },
+                clearInput: async (target)=>{
+                    node_assert(libnut, 'libnut not initialized');
+                    if (target) {
+                        const element = target;
+                        const [x, y] = element.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await sleep(100);
+                    }
+                    await this.selectAllAndDelete();
+                    await sleep(50);
+                }
+            },
+            scroll: {
+                scroll: async (param)=>{
+                    await this.performScroll(param);
+                }
+            }
+        });
         this.options = options;
         this.displayId = options?.displayId;
         this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -1171,132 +1144,7 @@ class RDPDevice {
     }
     actionSpace() {
         const defaultActions = [
-            defineActionTap(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'tap');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'down');
-                await sleep(device_CLICK_HOLD_DURATION);
-                await this.backend.mouseButton('left', 'up');
-            }),
-            defineActionDoubleClick(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'double click');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'doubleClick');
-            }),
-            defineActionRightClick(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'right click');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('right', 'click');
-            }),
-            defineActionHover(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'hover');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
-                    settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
-                });
-            }),
-            defineActionInput(async (param)=>{
-                this.assertConnected();
-                if (param.locate) {
-                    await this.moveToElement(param.locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                    await sleep(device_INPUT_FOCUS_DELAY);
-                }
-                if ('typeOnly' !== param.mode) {
-                    await this.clearInput();
-                    await sleep(device_INPUT_CLEAR_DELAY);
-                }
-                if ('clear' === param.mode) return;
-                if (param.value) await this.backend.typeText(param.value);
-            }),
-            defineActionClearInput(async ({ locate })=>{
-                this.assertConnected();
-                if (locate) {
-                    await this.moveToElement(locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                    await sleep(device_INPUT_FOCUS_DELAY);
-                }
-                await this.clearInput();
-                await sleep(device_INPUT_CLEAR_DELAY);
-            }),
-            defineActionKeyboardPress(async ({ locate, keyName })=>{
-                this.assertConnected();
-                if (locate) {
-                    await this.moveToElement(locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                }
-                await this.backend.keyPress(keyName);
-            }),
-            defineActionScroll(async (param)=>{
-                this.assertConnected();
-                const target = param.locate;
-                if (target) await this.moveToElement(target, {
-                    steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
-                });
-                if (param.scrollType && 'singleAction' !== param.scrollType) {
-                    const direction = this.edgeScrollDirection(param.scrollType);
-                    for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
-                    await sleep(device_SCROLL_COMPLETE_DELAY);
-                    return;
-                }
-                await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
-                await sleep(device_SCROLL_COMPLETE_DELAY);
-            }),
-            defineActionDragAndDrop(async ({ from, to })=>{
-                this.assertConnected();
-                const source = this.requireLocate(from, 'drag source');
-                const target = this.requireLocate(to, 'drag target');
-                await this.moveToElement(source, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'down');
-                await sleep(DRAG_HOLD_DURATION);
-                await this.moveToElement(target, {
-                    steps: SMOOTH_MOVE_STEPS_DRAG,
-                    stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
-                });
-                await sleep(DRAG_HOLD_DURATION);
-                await this.backend.mouseButton('left', 'up');
-            }),
-            defineAction({
-                name: 'MiddleClick',
-                description: 'Middle click the element',
-                sample: {
-                    locate: {
-                        prompt: 'the browser tab close target'
-                    }
-                },
-                paramSchema: actionTapParamSchema,
-                call: async ({ locate })=>{
-                    const element = this.requireLocate(locate, 'middle click');
-                    await this.moveToElement(element, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('middle', 'click');
-                }
-            }),
+            ...defineActionsFromInputPrimitives(this.inputPrimitives),
             defineAction({
                 name: 'ListDisplays',
                 description: 'List all available displays/monitors',
@@ -1325,10 +1173,6 @@ class RDPDevice {
     throwIfDestroyed() {
         if (this.destroyed) throw new Error('RDPDevice has been destroyed');
     }
-    requireLocate(locate, actionName) {
-        if (!locate) throw new Error(`Missing target element for ${actionName}`);
-        return locate;
-    }
     async moveToElement(element, options) {
         this.assertConnected();
         const targetX = Math.round(element.center[0]);
@@ -1394,6 +1238,113 @@ class RDPDevice {
         device_define_property(this, "destroyed", false);
         device_define_property(this, "cursorPosition", void 0);
         device_define_property(this, "uri", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'down');
+                    await sleep(device_CLICK_HOLD_DURATION);
+                    await this.backend.mouseButton('left', 'up');
+                },
+                doubleClick: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'doubleClick');
+                },
+                rightClick: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('right', 'click');
+                },
+                hover: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
+                        settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
+                    });
+                },
+                dragAndDrop: async (from, to)=>{
+                    await this.movePointer(Math.round(from.x), Math.round(from.y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'down');
+                    await sleep(DRAG_HOLD_DURATION);
+                    await this.movePointer(Math.round(to.x), Math.round(to.y), {
+                        steps: SMOOTH_MOVE_STEPS_DRAG,
+                        stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
+                    });
+                    await sleep(DRAG_HOLD_DURATION);
+                    await this.backend.mouseButton('left', 'up');
+                }
+            },
+            keyboard: {
+                typeText: async (value, opts)=>{
+                    this.assertConnected();
+                    const target = opts?.target;
+                    if (target) {
+                        await this.inputPrimitives.pointer.tap({
+                            x: target.center[0],
+                            y: target.center[1]
+                        });
+                        await sleep(device_INPUT_FOCUS_DELAY);
+                    }
+                    if (opts?.replace !== false) {
+                        await this.clearInput();
+                        await sleep(device_INPUT_CLEAR_DELAY);
+                    }
+                    if (opts?.focusOnly || !value) return;
+                    await this.backend.typeText(value);
+                },
+                clearInput: async (target)=>{
+                    this.assertConnected();
+                    const element = target;
+                    if (element) {
+                        await this.inputPrimitives.pointer.tap({
+                            x: element.center[0],
+                            y: element.center[1]
+                        });
+                        await sleep(device_INPUT_FOCUS_DELAY);
+                    }
+                    await this.clearInput();
+                    await sleep(device_INPUT_CLEAR_DELAY);
+                },
+                keyboardPress: async (keyName, opts)=>{
+                    this.assertConnected();
+                    const target = opts?.target;
+                    if (target) await this.inputPrimitives.pointer.tap({
+                        x: target.center[0],
+                        y: target.center[1]
+                    });
+                    await this.backend.keyPress(keyName);
+                }
+            },
+            scroll: {
+                scroll: async (param)=>{
+                    this.assertConnected();
+                    const target = param.locate;
+                    if (target) await this.moveToElement(target, {
+                        steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
+                    });
+                    if (param.scrollType && 'singleAction' !== param.scrollType) {
+                        const direction = this.edgeScrollDirection(param.scrollType);
+                        for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
+                        await sleep(device_SCROLL_COMPLETE_DELAY);
+                        return;
+                    }
+                    await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
+                    await sleep(device_SCROLL_COMPLETE_DELAY);
+                }
+            }
+        });
         this.options = {
             port: 3389,
             securityProtocol: 'auto',
@@ -1625,7 +1576,7 @@ class ComputerMCPServer extends BaseMCPServer {
     constructor(toolsManager){
         super({
             name: '@midscene/computer-mcp',
-            version: "1.8.1-beta-20260513084557.0",
+            version: "1.8.1",
             description: 'Control the computer desktop using natural language commands'
         }, toolsManager);
     }