@midscene/computer 1.8.1-beta-20260513084557.0 → 1.8.1

package/dist/lib/mcp-server.js

@@ -49,7 +49,6 @@ const external_node_fs_namespaceObject = require("node:fs");
 const external_node_module_namespaceObject = require("node:module");
 const external_node_path_namespaceObject = require("node:path");
 const external_node_url_namespaceObject = require("node:url");
-const core_namespaceObject = require("@midscene/core");
 const device_namespaceObject = require("@midscene/core/device");
 const utils_namespaceObject = require("@midscene/core/utils");
 const img_namespaceObject = require("@midscene/shared/img");
@@ -133,15 +132,6 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const computerInputParamSchema = core_namespaceObject.z.object({
-    value: core_namespaceObject.z.string().describe('The text to input'),
-    mode: core_namespaceObject.z["enum"]([
-        'replace',
-        'clear',
-        'append'
-    ]).default('replace').optional().describe('Input mode: replace, clear, or append'),
-    locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
-});
 const SMOOTH_MOVE_STEPS_TAP = 8;
 const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
 const SMOOTH_MOVE_DELAY_TAP = 8;
@@ -452,7 +442,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck() {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.8.1-beta-20260513084557.0");
+        console.log("[HealthCheck] @midscene/computer v1.8.1");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -518,21 +508,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         debugDevice('Taking screenshot', {
             displayId: this.displayId
         });
-        try {
-            const options = {
-                format: 'png'
-            };
-            if (void 0 !== this.displayId) if ('darwin' === process.platform) {
-                const screenIndex = Number(this.displayId);
-                if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
-            } else options.screen = this.displayId;
-            debugDevice('Screenshot options', options);
+        const options = {
+            format: 'png'
+        };
+        if (void 0 !== this.displayId) if ('darwin' === process.platform) {
+            const screenIndex = Number(this.displayId);
+            if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
+        } else options.screen = this.displayId;
+        debugDevice('Screenshot options', options);
+        const MAX_ATTEMPTS = 3;
+        const RETRY_DELAY_MS = 300;
+        let lastRawMessage = '';
+        for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
             const buffer = await external_screenshot_desktop_default()(options);
+            if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
             return (0, img_namespaceObject.createImgBase64ByFormat)('png', buffer.toString('base64'));
         } catch (error) {
-            debugDevice(`Screenshot failed: ${error}`);
-            throw new Error(`Failed to take screenshot: ${error}`);
+            lastRawMessage = error instanceof Error ? error.message : String(error);
+            const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
+            const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
+            debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
+            if (!willRetry) break;
+            await (0, utils_namespaceObject.sleep)(RETRY_DELAY_MS);
         }
+        if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
+
+Please follow these steps:
+1. Open System Settings > Privacy & Security > Screen Recording
+2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
+3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
+
+Original error: ${lastRawMessage}`);
+        throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
     }
     async size() {
         external_node_assert_default()(libnut, 'libnut not initialized');
@@ -578,228 +585,111 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         external_node_assert_default()(libnut, 'libnut not initialized');
         await this.typeViaClipboard(text);
     }
+    async selectAllAndDelete() {
+        external_node_assert_default()(libnut, 'libnut not initialized');
+        if (this.useAppleScript) {
+            sendKeyViaAppleScript('a', [
+                'command'
+            ]);
+            await (0, utils_namespaceObject.sleep)(50);
+            sendKeyViaAppleScript('backspace', []);
+            return;
+        }
+        const modifier = 'darwin' === process.platform ? 'command' : 'control';
+        libnut.keyTap('a', [
+            modifier
+        ]);
+        await (0, utils_namespaceObject.sleep)(50);
+        libnut.keyTap('backspace');
+    }
+    async pressKeyboardShortcut(keyName) {
+        external_node_assert_default()(libnut, 'libnut not initialized');
+        const keys = keyName.split('+');
+        const modifiers = keys.slice(0, -1).map(normalizeKeyName);
+        const key = normalizePrimaryKey(keys[keys.length - 1]);
+        debugDevice('KeyboardPress', {
+            original: keyName,
+            key,
+            modifiers,
+            driver: this.useAppleScript ? "applescript" : 'libnut'
+        });
+        if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
+        else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
+        else libnut.keyTap(key);
+    }
+    async performScroll(param) {
+        external_node_assert_default()(libnut, 'libnut not initialized');
+        if (param.locate) {
+            const element = param.locate;
+            const [x, y] = element.center;
+            libnut.moveMouse(Math.round(x), Math.round(y));
+        }
+        const scrollType = param?.scrollType;
+        const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
+        if (edgeSpec) {
+            if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
+            if (this.useAppleScript) {
+                sendKeyViaAppleScript(edgeSpec.key);
+                await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
+                return;
+            }
+            const [dx, dy] = edgeSpec.libnut;
+            for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
+                libnut.scrollMouse(dx, dy);
+                await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
+            }
+            return;
+        }
+        if ('singleAction' === scrollType || !scrollType) {
+            const distance = param?.distance || 500;
+            const direction = param?.direction || 'down';
+            const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
+            if (isKnownDirection) {
+                const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
+                if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
+            }
+            if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
+                const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
+                const key = 'up' === direction ? 'pageup' : 'pagedown';
+                for(let i = 0; i < pages; i++){
+                    sendKeyViaAppleScript(key);
+                    await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
+                }
+                await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
+                return;
+            }
+            const ticks = Math.ceil(distance / 100);
+            const directionMap = {
+                up: [
+                    0,
+                    ticks
+                ],
+                down: [
+                    0,
+                    -ticks
+                ],
+                left: [
+                    -ticks,
+                    0
+                ],
+                right: [
+                    ticks,
+                    0
+                ]
+            };
+            const [dx, dy] = directionMap[direction] || [
+                0,
+                -ticks
+            ];
+            libnut.scrollMouse(dx, dy);
+            await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
+            return;
+        }
+        throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
+    }
     actionSpace() {
         const defaultActions = [
-            (0, device_namespaceObject.defineActionTap)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot tap');
-                const [x, y] = element.center;
-                const targetX = Math.round(x);
-                const targetY = Math.round(y);
-                await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
-                libnut.mouseToggle('down', 'left');
-                await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
-                libnut.mouseToggle('up', 'left');
-            }),
-            (0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot double click');
-                const [x, y] = element.center;
-                libnut.moveMouse(Math.round(x), Math.round(y));
-                libnut.mouseClick('left', true);
-            }),
-            (0, device_namespaceObject.defineActionRightClick)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot right click');
-                const [x, y] = element.center;
-                libnut.moveMouse(Math.round(x), Math.round(y));
-                libnut.mouseClick('right');
-            }),
-            (0, device_namespaceObject.defineAction)({
-                name: 'MouseMove',
-                description: 'Move the mouse to the element',
-                interfaceAlias: 'aiHover',
-                paramSchema: device_namespaceObject.actionHoverParamSchema,
-                sample: {
-                    locate: {
-                        prompt: 'the navigation menu item "Products"'
-                    }
-                },
-                call: async (param)=>{
-                    external_node_assert_default()(libnut, 'libnut not initialized');
-                    const element = param.locate;
-                    external_node_assert_default()(element, 'Element not found, cannot move mouse');
-                    const [x, y] = element.center;
-                    const targetX = Math.round(x);
-                    const targetY = Math.round(y);
-                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
-                    await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
-                }
-            }),
-            (0, device_namespaceObject.defineAction)({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: computerInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    external_node_assert_default()(libnut, 'libnut not initialized');
-                    const element = param.locate;
-                    if (element) {
-                        const [x, y] = element.center;
-                        libnut.moveMouse(Math.round(x), Math.round(y));
-                        libnut.mouseClick('left');
-                        await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
-                        if ('append' !== param.mode) {
-                            if (this.useAppleScript) {
-                                sendKeyViaAppleScript('a', [
-                                    'command'
-                                ]);
-                                await (0, utils_namespaceObject.sleep)(50);
-                                sendKeyViaAppleScript('backspace', []);
-                            } else {
-                                const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                                libnut.keyTap('a', [
-                                    modifier
-                                ]);
-                                await (0, utils_namespaceObject.sleep)(50);
-                                libnut.keyTap('backspace');
-                            }
-                            await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
-                        }
-                    }
-                    if ('clear' === param.mode) return;
-                    if (!param.value) return;
-                    await this.smartTypeString(param.value);
-                }
-            }),
-            (0, device_namespaceObject.defineActionScroll)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                if (param.locate) {
-                    const element = param.locate;
-                    const [x, y] = element.center;
-                    libnut.moveMouse(Math.round(x), Math.round(y));
-                }
-                const scrollType = param?.scrollType;
-                const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
-                if (edgeSpec) {
-                    if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
-                    if (this.useAppleScript) {
-                        sendKeyViaAppleScript(edgeSpec.key);
-                        await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
-                        return;
-                    }
-                    const [dx, dy] = edgeSpec.libnut;
-                    for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
-                        libnut.scrollMouse(dx, dy);
-                        await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
-                    }
-                    return;
-                }
-                if ('singleAction' === scrollType || !scrollType) {
-                    const distance = param?.distance || 500;
-                    const direction = param?.direction || 'down';
-                    const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
-                    if (isKnownDirection) {
-                        const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
-                        if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
-                    }
-                    if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
-                        const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
-                        const key = 'up' === direction ? 'pageup' : 'pagedown';
-                        for(let i = 0; i < pages; i++){
-                            sendKeyViaAppleScript(key);
-                            await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
-                        }
-                        await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
-                        return;
-                    }
-                    const ticks = Math.ceil(distance / 100);
-                    const directionMap = {
-                        up: [
-                            0,
-                            ticks
-                        ],
-                        down: [
-                            0,
-                            -ticks
-                        ],
-                        left: [
-                            -ticks,
-                            0
-                        ],
-                        right: [
-                            ticks,
-                            0
-                        ]
-                    };
-                    const [dx, dy] = directionMap[direction] || [
-                        0,
-                        -ticks
-                    ];
-                    libnut.scrollMouse(dx, dy);
-                    await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
-                    return;
-                }
-                throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
-            }),
-            (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                if (param.locate) {
-                    const [x, y] = param.locate.center;
-                    libnut.moveMouse(Math.round(x), Math.round(y));
-                    libnut.mouseClick('left');
-                    await (0, utils_namespaceObject.sleep)(50);
-                }
-                const keys = param.keyName.split('+');
-                const modifiers = keys.slice(0, -1).map(normalizeKeyName);
-                const key = normalizePrimaryKey(keys[keys.length - 1]);
-                debugDevice('KeyboardPress', {
-                    original: param.keyName,
-                    key,
-                    modifiers,
-                    driver: this.useAppleScript ? "applescript" : 'libnut'
-                });
-                if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
-                else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
-                else libnut.keyTap(key);
-            }),
-            (0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                const from = param.from;
-                const to = param.to;
-                external_node_assert_default()(from, 'missing "from" param for drag and drop');
-                external_node_assert_default()(to, 'missing "to" param for drag and drop');
-                const [fromX, fromY] = from.center;
-                const [toX, toY] = to.center;
-                libnut.moveMouse(Math.round(fromX), Math.round(fromY));
-                libnut.mouseToggle('down', 'left');
-                await (0, utils_namespaceObject.sleep)(100);
-                libnut.moveMouse(Math.round(toX), Math.round(toY));
-                await (0, utils_namespaceObject.sleep)(100);
-                libnut.mouseToggle('up', 'left');
-            }),
-            (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
-                external_node_assert_default()(libnut, 'libnut not initialized');
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot clear input');
-                const [x, y] = element.center;
-                libnut.moveMouse(Math.round(x), Math.round(y));
-                libnut.mouseClick('left');
-                await (0, utils_namespaceObject.sleep)(100);
-                if (this.useAppleScript) {
-                    sendKeyViaAppleScript('a', [
-                        'command'
-                    ]);
-                    await (0, utils_namespaceObject.sleep)(50);
-                    sendKeyViaAppleScript('backspace', []);
-                } else {
-                    const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                    libnut.keyTap('a', [
-                        modifier
-                    ]);
-                    libnut.keyTap('backspace');
-                }
-                await (0, utils_namespaceObject.sleep)(50);
-            })
+            ...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives)
         ];
         const platformActions = Object.values(createPlatformActions());
         const customActions = this.options?.customActions || [];
@@ -837,6 +727,88 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         _define_property(this, "xvfbCleanup", void 0);
         _define_property(this, "useAppleScript", void 0);
         _define_property(this, "uri", void 0);
+        _define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: async ({ x, y })=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    const targetX = Math.round(x);
+                    const targetY = Math.round(y);
+                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
+                    libnut.mouseToggle('down', 'left');
+                    await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
+                    libnut.mouseToggle('up', 'left');
+                },
+                doubleClick: async ({ x, y })=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    libnut.moveMouse(Math.round(x), Math.round(y));
+                    libnut.mouseClick('left', true);
+                },
+                rightClick: async ({ x, y })=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    libnut.moveMouse(Math.round(x), Math.round(y));
+                    libnut.mouseClick('right');
+                },
+                hover: async ({ x, y })=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    await smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
+                    await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
+                },
+                dragAndDrop: async (from, to)=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    libnut.moveMouse(Math.round(from.x), Math.round(from.y));
+                    libnut.mouseToggle('down', 'left');
+                    await (0, utils_namespaceObject.sleep)(100);
+                    libnut.moveMouse(Math.round(to.x), Math.round(to.y));
+                    await (0, utils_namespaceObject.sleep)(100);
+                    libnut.mouseToggle('up', 'left');
+                }
+            },
+            keyboard: {
+                typeText: async (value, opts)=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    const element = opts?.target;
+                    if (element) {
+                        const [x, y] = element.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
+                        if (opts?.replace !== false) {
+                            await this.selectAllAndDelete();
+                            await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
+                        }
+                    }
+                    await this.smartTypeString(value);
+                },
+                keyboardPress: async (keyName, opts)=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    const target = opts?.target;
+                    if (target) {
+                        const [x, y] = target.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await (0, utils_namespaceObject.sleep)(50);
+                    }
+                    await this.pressKeyboardShortcut(keyName);
+                },
+                clearInput: async (target)=>{
+                    external_node_assert_default()(libnut, 'libnut not initialized');
+                    if (target) {
+                        const element = target;
+                        const [x, y] = element.center;
+                        libnut.moveMouse(Math.round(x), Math.round(y));
+                        libnut.mouseClick('left');
+                        await (0, utils_namespaceObject.sleep)(100);
+                    }
+                    await this.selectAllAndDelete();
+                    await (0, utils_namespaceObject.sleep)(50);
+                }
+            },
+            scroll: {
+                scroll: async (param)=>{
+                    await this.performScroll(param);
+                }
+            }
+        });
         this.options = options;
         this.displayId = options?.displayId;
         this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -1213,132 +1185,7 @@ class RDPDevice {
     }
     actionSpace() {
         const defaultActions = [
-            (0, device_namespaceObject.defineActionTap)(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'tap');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'down');
-                await (0, utils_namespaceObject.sleep)(device_CLICK_HOLD_DURATION);
-                await this.backend.mouseButton('left', 'up');
-            }),
-            (0, device_namespaceObject.defineActionDoubleClick)(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'double click');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'doubleClick');
-            }),
-            (0, device_namespaceObject.defineActionRightClick)(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'right click');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('right', 'click');
-            }),
-            (0, device_namespaceObject.defineActionHover)(async ({ locate })=>{
-                const element = this.requireLocate(locate, 'hover');
-                await this.moveToElement(element, {
-                    steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
-                    settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
-                });
-            }),
-            (0, device_namespaceObject.defineActionInput)(async (param)=>{
-                this.assertConnected();
-                if (param.locate) {
-                    await this.moveToElement(param.locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                    await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
-                }
-                if ('typeOnly' !== param.mode) {
-                    await this.clearInput();
-                    await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
-                }
-                if ('clear' === param.mode) return;
-                if (param.value) await this.backend.typeText(param.value);
-            }),
-            (0, device_namespaceObject.defineActionClearInput)(async ({ locate })=>{
-                this.assertConnected();
-                if (locate) {
-                    await this.moveToElement(locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                    await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
-                }
-                await this.clearInput();
-                await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
-            }),
-            (0, device_namespaceObject.defineActionKeyboardPress)(async ({ locate, keyName })=>{
-                this.assertConnected();
-                if (locate) {
-                    await this.moveToElement(locate, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('left', 'click');
-                }
-                await this.backend.keyPress(keyName);
-            }),
-            (0, device_namespaceObject.defineActionScroll)(async (param)=>{
-                this.assertConnected();
-                const target = param.locate;
-                if (target) await this.moveToElement(target, {
-                    steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
-                });
-                if (param.scrollType && 'singleAction' !== param.scrollType) {
-                    const direction = this.edgeScrollDirection(param.scrollType);
-                    for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
-                    await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
-                    return;
-                }
-                await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
-                await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
-            }),
-            (0, device_namespaceObject.defineActionDragAndDrop)(async ({ from, to })=>{
-                this.assertConnected();
-                const source = this.requireLocate(from, 'drag source');
-                const target = this.requireLocate(to, 'drag target');
-                await this.moveToElement(source, {
-                    steps: device_SMOOTH_MOVE_STEPS_TAP,
-                    stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                });
-                await this.backend.mouseButton('left', 'down');
-                await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
-                await this.moveToElement(target, {
-                    steps: SMOOTH_MOVE_STEPS_DRAG,
-                    stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
-                });
-                await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
-                await this.backend.mouseButton('left', 'up');
-            }),
-            (0, device_namespaceObject.defineAction)({
-                name: 'MiddleClick',
-                description: 'Middle click the element',
-                sample: {
-                    locate: {
-                        prompt: 'the browser tab close target'
-                    }
-                },
-                paramSchema: device_namespaceObject.actionTapParamSchema,
-                call: async ({ locate })=>{
-                    const element = this.requireLocate(locate, 'middle click');
-                    await this.moveToElement(element, {
-                        steps: device_SMOOTH_MOVE_STEPS_TAP,
-                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
-                    });
-                    await this.backend.mouseButton('middle', 'click');
-                }
-            }),
+            ...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives),
             (0, device_namespaceObject.defineAction)({
                 name: 'ListDisplays',
                 description: 'List all available displays/monitors',
@@ -1367,10 +1214,6 @@ class RDPDevice {
     throwIfDestroyed() {
         if (this.destroyed) throw new Error('RDPDevice has been destroyed');
     }
-    requireLocate(locate, actionName) {
-        if (!locate) throw new Error(`Missing target element for ${actionName}`);
-        return locate;
-    }
     async moveToElement(element, options) {
         this.assertConnected();
         const targetX = Math.round(element.center[0]);
@@ -1436,6 +1279,113 @@ class RDPDevice {
         device_define_property(this, "destroyed", false);
         device_define_property(this, "cursorPosition", void 0);
         device_define_property(this, "uri", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'down');
+                    await (0, utils_namespaceObject.sleep)(device_CLICK_HOLD_DURATION);
+                    await this.backend.mouseButton('left', 'up');
+                },
+                doubleClick: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'doubleClick');
+                },
+                rightClick: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('right', 'click');
+                },
+                hover: async ({ x, y })=>{
+                    await this.movePointer(Math.round(x), Math.round(y), {
+                        steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
+                        settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
+                    });
+                },
+                dragAndDrop: async (from, to)=>{
+                    await this.movePointer(Math.round(from.x), Math.round(from.y), {
+                        steps: device_SMOOTH_MOVE_STEPS_TAP,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
+                    });
+                    await this.backend.mouseButton('left', 'down');
+                    await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
+                    await this.movePointer(Math.round(to.x), Math.round(to.y), {
+                        steps: SMOOTH_MOVE_STEPS_DRAG,
+                        stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
+                    });
+                    await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
+                    await this.backend.mouseButton('left', 'up');
+                }
+            },
+            keyboard: {
+                typeText: async (value, opts)=>{
+                    this.assertConnected();
+                    const target = opts?.target;
+                    if (target) {
+                        await this.inputPrimitives.pointer.tap({
+                            x: target.center[0],
+                            y: target.center[1]
+                        });
+                        await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
+                    }
+                    if (opts?.replace !== false) {
+                        await this.clearInput();
+                        await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
+                    }
+                    if (opts?.focusOnly || !value) return;
+                    await this.backend.typeText(value);
+                },
+                clearInput: async (target)=>{
+                    this.assertConnected();
+                    const element = target;
+                    if (element) {
+                        await this.inputPrimitives.pointer.tap({
+                            x: element.center[0],
+                            y: element.center[1]
+                        });
+                        await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
+                    }
+                    await this.clearInput();
+                    await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
+                },
+                keyboardPress: async (keyName, opts)=>{
+                    this.assertConnected();
+                    const target = opts?.target;
+                    if (target) await this.inputPrimitives.pointer.tap({
+                        x: target.center[0],
+                        y: target.center[1]
+                    });
+                    await this.backend.keyPress(keyName);
+                }
+            },
+            scroll: {
+                scroll: async (param)=>{
+                    this.assertConnected();
+                    const target = param.locate;
+                    if (target) await this.moveToElement(target, {
+                        steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
+                        stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
+                    });
+                    if (param.scrollType && 'singleAction' !== param.scrollType) {
+                        const direction = this.edgeScrollDirection(param.scrollType);
+                        for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
+                        await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
+                        return;
+                    }
+                    await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
+                    await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
+                }
+            }
+        });
         this.options = {
             port: 3389,
             securityProtocol: 'auto',
@@ -1483,6 +1433,7 @@ async function agentForRDPComputer(opts) {
     await device.connect();
     return new ComputerAgent(device, opts);
 }
+const core_namespaceObject = require("@midscene/core");
 const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
 function mcp_tools_define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -1668,7 +1619,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
     constructor(toolsManager){
         super({
             name: '@midscene/computer-mcp',
-            version: "1.8.1-beta-20260513084557.0",
+            version: "1.8.1",
             description: 'Control the computer desktop using natural language commands'
         }, toolsManager);
     }