@midscene/computer 1.8.6 → 1.8.7-beta-20260527113633.0

package/dist/es/index.mjs

@@ -1,4 +1,3 @@
-import node_assert from "node:assert";
 import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
@@ -9,12 +8,141 @@ import { sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import screenshot_desktop from "screenshot-desktop";
+import node_assert from "node:assert";
 import { Agent } from "@midscene/core/agent";
 import { once } from "node:events";
 import { createInterface } from "node:readline";
 import { z } from "@midscene/core";
 import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
 import { overrideAIConfig } from "@midscene/shared/env";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class ComputerInputDriver {
+    destroy() {
+        if (this.destroyed) return;
+        this.destroyed = true;
+        this.rejectPendingInputDelays();
+    }
+    getScreenSize() {
+        return this.getLibnutOrThrow('getScreenSize').getScreenSize();
+    }
+    getMousePos() {
+        return this.getLibnutOrThrow('getMousePos').getMousePos();
+    }
+    moveMouse(x, y) {
+        this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
+    }
+    mouseClick(button, double) {
+        const lib = this.getLibnutOrThrow('mouseClick');
+        if (void 0 !== double) lib.mouseClick(button, double);
+        else if (void 0 !== button) lib.mouseClick(button);
+        else lib.mouseClick();
+    }
+    mouseToggle(state, button = 'left') {
+        this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
+    }
+    scrollMouse(x, y) {
+        this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
+    }
+    keyTap(key, modifiers) {
+        const lib = this.getLibnutOrThrow('keyTap');
+        if (void 0 !== modifiers) lib.keyTap(key, modifiers);
+        else lib.keyTap(key);
+    }
+    sendKeyViaAppleScript(key, modifiers = []) {
+        this.assertActive('sendKeyViaAppleScript');
+        this.options.sendKeyViaAppleScript(key, modifiers);
+    }
+    sendKey(key, modifiers = []) {
+        if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
+        if (modifiers.length > 0) this.keyTap(key, modifiers);
+        else this.keyTap(key);
+    }
+    runPhasedScroll(direction, pixels, steps) {
+        this.assertActive('runPhasedScroll');
+        return this.options.runPhasedScroll(direction, pixels, steps);
+    }
+    async delay(ms) {
+        this.assertActive('delay');
+        return new Promise((resolve, reject)=>{
+            const waitRef = {
+                timeoutId: setTimeout(()=>{
+                    this.pendingInputDelayWaits.delete(waitRef);
+                    try {
+                        this.assertActive('delay');
+                        resolve();
+                    } catch (error) {
+                        reject(error);
+                    }
+                }, ms),
+                reject
+            };
+            this.pendingInputDelayWaits.add(waitRef);
+        });
+    }
+    async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
+        const currentPos = this.getMousePos();
+        for(let i = 1; i <= steps; i++){
+            const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
+            const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
+            this.moveMouse(stepX, stepY);
+            await this.delay(stepDelay);
+        }
+    }
+    async withMouseButton(button, run) {
+        this.mouseToggle('down', button);
+        try {
+            return await run();
+        } finally{
+            this.releaseMouseButton(button);
+        }
+    }
+    getLibnutOrThrow(methodName) {
+        this.assertActive(methodName);
+        const libnut = this.options.getLibnut();
+        node_assert(libnut, 'libnut not initialized');
+        return libnut;
+    }
+    assertActive(methodName) {
+        if (this.destroyed) throw this.createDestroyedError(methodName);
+    }
+    createDestroyedError(methodName) {
+        return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
+    }
+    releaseMouseButton(button) {
+        try {
+            const libnut = this.options.getLibnut();
+            node_assert(libnut, 'libnut not initialized');
+            libnut.mouseToggle('up', button);
+        } catch (error) {
+            this.options.debug(`Failed to release mouse button ${button}: ${error}`);
+        }
+    }
+    rejectPendingInputDelays() {
+        const error = this.createDestroyedError('in-flight input');
+        for (const waitRef of this.pendingInputDelayWaits){
+            clearTimeout(waitRef.timeoutId);
+            waitRef.reject(error);
+        }
+        this.pendingInputDelayWaits.clear();
+    }
+    constructor(options){
+        _define_property(this, "options", void 0);
+        _define_property(this, "destroyed", void 0);
+        _define_property(this, "pendingInputDelayWaits", void 0);
+        this.options = options;
+        this.destroyed = false;
+        this.pendingInputDelayWaits = new Set();
+    }
+}
 const debugXvfb = getDebug('computer:xvfb');
 function checkXvfbInstalled() {
     try {
@@ -82,7 +210,7 @@ function needsXvfb(explicitOpt) {
     if ('linux' !== process.platform) return false;
     return true === explicitOpt;
 }
-function _define_property(obj, key, value) {
+function device_define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
         enumerable: true,
@@ -283,16 +411,6 @@ function runPhasedScroll(direction, pixels, steps) {
         return false;
     }
 }
-async function smoothMoveMouse(targetX, targetY, steps, stepDelay) {
-    node_assert(device_libnut, 'libnut not initialized');
-    const currentPos = device_libnut.getMousePos();
-    for(let i = 1; i <= steps; i++){
-        const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
-        const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
-        device_libnut.moveMouse(stepX, stepY);
-        await sleep(stepDelay);
-    }
-}
 const KEY_NAME_MAP = {
     windows: 'win',
     win: 'win',
@@ -402,7 +520,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck() {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.8.6");
+        console.log("[HealthCheck] @midscene/computer v1.8.7-beta-20260527113633.0");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -415,17 +533,16 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         ]);
         console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
         console.log('[HealthCheck] Moving mouse...');
-        node_assert(device_libnut, 'libnut not initialized');
-        const startPos = device_libnut.getMousePos();
+        const startPos = this.inputDriver.getMousePos();
         console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
         const offsetX = Math.floor(40 * Math.random()) + 10;
         const offsetY = Math.floor(40 * Math.random()) + 10;
         const targetX = startPos.x + offsetX;
         const targetY = startPos.y + offsetY;
         console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
-        device_libnut.moveMouse(targetX, targetY);
+        this.inputDriver.moveMouse(targetX, targetY);
         await sleep(50);
-        const movedPos = device_libnut.getMousePos();
+        const movedPos = this.inputDriver.getMousePos();
         console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
         const deltaX = Math.abs(movedPos.x - targetX);
         const deltaY = Math.abs(movedPos.y - targetY);
@@ -439,7 +556,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
                 debugDevice(hint);
             }
         }
-        device_libnut.moveMouse(startPos.x, startPos.y);
+        this.inputDriver.moveMouse(startPos.x, startPos.y);
         console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
         console.log('[HealthCheck] Listing monitors...');
         const displays = await ComputerDevice.listDisplays();
@@ -502,9 +619,8 @@ Original error: ${lastRawMessage}`);
         throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
     }
     async size() {
-        node_assert(device_libnut, 'libnut not initialized');
         try {
-            const screenSize = device_libnut.getScreenSize();
+            const screenSize = this.inputDriver.getScreenSize();
             return {
                 width: screenSize.width,
                 height: screenSize.height
@@ -515,7 +631,6 @@ Original error: ${lastRawMessage}`);
         }
     }
     async typeViaClipboard(text) {
-        node_assert(device_libnut, 'libnut not initialized');
         debugDevice('Using clipboard to input text', {
             textLength: text.length,
             preview: text.substring(0, 20)
@@ -524,17 +639,17 @@ Original error: ${lastRawMessage}`);
         const oldClipboard = await clipboardy.default.read().catch(()=>'');
         try {
             await clipboardy.default.write(text);
-            await sleep(50);
-            if (this.useAppleScript) sendKeyViaAppleScript('v', [
+            await this.inputDriver.delay(50);
+            if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
                 'command'
             ]);
             else {
                 const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                device_libnut.keyTap('v', [
+                this.inputDriver.keyTap('v', [
                     modifier
                 ]);
             }
-            await sleep(100);
+            await this.inputDriver.delay(100);
         } finally{
             if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
                 debugDevice('Failed to restore clipboard content');
@@ -542,28 +657,25 @@ Original error: ${lastRawMessage}`);
         }
     }
     async smartTypeString(text) {
-        node_assert(device_libnut, 'libnut not initialized');
         await this.typeViaClipboard(text);
     }
     async selectAllAndDelete() {
-        node_assert(device_libnut, 'libnut not initialized');
         if (this.useAppleScript) {
-            sendKeyViaAppleScript('a', [
+            this.inputDriver.sendKeyViaAppleScript('a', [
                 'command'
             ]);
-            await sleep(50);
-            sendKeyViaAppleScript('backspace', []);
+            await this.inputDriver.delay(50);
+            this.inputDriver.sendKeyViaAppleScript('backspace', []);
             return;
         }
         const modifier = 'darwin' === process.platform ? 'command' : 'control';
-        device_libnut.keyTap('a', [
+        this.inputDriver.keyTap('a', [
             modifier
         ]);
-        await sleep(50);
-        device_libnut.keyTap('backspace');
+        await this.inputDriver.delay(50);
+        this.inputDriver.keyTap('backspace');
     }
     async pressKeyboardShortcut(keyName) {
-        node_assert(device_libnut, 'libnut not initialized');
         const keys = keyName.split('+');
         const modifiers = keys.slice(0, -1).map(normalizeKeyName);
         const key = normalizePrimaryKey(keys[keys.length - 1]);
@@ -573,30 +685,27 @@ Original error: ${lastRawMessage}`);
             modifiers,
             driver: this.useAppleScript ? "applescript" : 'libnut'
         });
-        if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
-        else if (modifiers.length > 0) device_libnut.keyTap(key, modifiers);
-        else device_libnut.keyTap(key);
+        this.inputDriver.sendKey(key, modifiers);
     }
     async performScroll(param) {
-        node_assert(device_libnut, 'libnut not initialized');
         if (param.locate) {
             const element = param.locate;
             const [x, y] = element.center;
-            device_libnut.moveMouse(Math.round(x), Math.round(y));
+            this.inputDriver.moveMouse(Math.round(x), Math.round(y));
         }
         const scrollType = param?.scrollType;
         const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
         if (edgeSpec) {
-            if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await sleep(SCROLL_COMPLETE_DELAY);
+            if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
             if (this.useAppleScript) {
-                sendKeyViaAppleScript(edgeSpec.key);
-                await sleep(SCROLL_COMPLETE_DELAY);
+                this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
+                await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
                 return;
             }
             const [dx, dy] = edgeSpec.libnut;
             for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
-                device_libnut.scrollMouse(dx, dy);
-                await sleep(SCROLL_STEP_DELAY);
+                this.inputDriver.scrollMouse(dx, dy);
+                await this.inputDriver.delay(SCROLL_STEP_DELAY);
             }
             return;
         }
@@ -613,17 +722,17 @@ Original error: ${lastRawMessage}`);
             }
             if (isKnownDirection) {
                 const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
-                if (runPhasedScroll(direction, distance, steps)) return void await sleep(SCROLL_COMPLETE_DELAY);
+                if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
             }
             if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
                 if (!screenSize) screenSize = await this.size();
                 const pages = Math.max(1, Math.round(distance / screenSize.height));
                 const key = 'up' === direction ? 'pageup' : 'pagedown';
                 for(let i = 0; i < pages; i++){
-                    sendKeyViaAppleScript(key);
-                    await sleep(SCROLL_STEP_DELAY);
+                    this.inputDriver.sendKeyViaAppleScript(key);
+                    await this.inputDriver.delay(SCROLL_STEP_DELAY);
                 }
-                await sleep(SCROLL_COMPLETE_DELAY);
+                await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
                 return;
             }
             const ticks = Math.ceil(distance / 100);
@@ -649,8 +758,8 @@ Original error: ${lastRawMessage}`);
                 0,
                 -ticks
             ];
-            device_libnut.scrollMouse(dx, dy);
-            await sleep(SCROLL_COMPLETE_DELAY);
+            this.inputDriver.scrollMouse(dx, dy);
+            await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
             return;
         }
         throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
@@ -669,6 +778,8 @@ Original error: ${lastRawMessage}`);
     }
     async destroy() {
         if (this.destroyed) return;
+        this.destroyed = true;
+        this.inputDriver.destroy();
         if (this.xvfbInstance) {
             this.xvfbInstance.stop();
             this.xvfbInstance = void 0;
@@ -679,96 +790,94 @@ Original error: ${lastRawMessage}`);
             process.removeListener('SIGTERM', this.xvfbCleanup);
             this.xvfbCleanup = void 0;
         }
-        this.destroyed = true;
         debugDevice('Computer device destroyed');
     }
     async url() {
         return '';
     }
     constructor(options){
-        _define_property(this, "interfaceType", 'computer');
-        _define_property(this, "options", void 0);
-        _define_property(this, "displayId", void 0);
-        _define_property(this, "description", void 0);
-        _define_property(this, "destroyed", false);
-        _define_property(this, "xvfbInstance", void 0);
-        _define_property(this, "xvfbCleanup", void 0);
-        _define_property(this, "useAppleScript", void 0);
-        _define_property(this, "uri", void 0);
-        _define_property(this, "inputPrimitives", {
+        device_define_property(this, "interfaceType", 'computer');
+        device_define_property(this, "options", void 0);
+        device_define_property(this, "displayId", void 0);
+        device_define_property(this, "description", void 0);
+        device_define_property(this, "destroyed", false);
+        device_define_property(this, "xvfbInstance", void 0);
+        device_define_property(this, "xvfbCleanup", void 0);
+        device_define_property(this, "inputDriver", new ComputerInputDriver({
+            getLibnut: ()=>device_libnut,
+            useAppleScript: ()=>this.useAppleScript,
+            sendKeyViaAppleScript,
+            runPhasedScroll,
+            debug: (message)=>debugDevice(message)
+        }));
+        device_define_property(this, "useAppleScript", void 0);
+        device_define_property(this, "uri", void 0);
+        device_define_property(this, "inputPrimitives", {
             pointer: {
                 tap: async ({ x, y })=>{
-                    node_assert(device_libnut, 'libnut not initialized');
                     const targetX = Math.round(x);
                     const targetY = Math.round(y);
-                    await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
-                    device_libnut.mouseToggle('down', 'left');
-                    await sleep(CLICK_HOLD_DURATION);
-                    device_libnut.mouseToggle('up', 'left');
+                    await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
+                    await this.inputDriver.withMouseButton('left', async ()=>{
+                        await this.inputDriver.delay(CLICK_HOLD_DURATION);
+                    });
                 },
                 doubleClick: async ({ x, y })=>{
-                    node_assert(device_libnut, 'libnut not initialized');
-                    device_libnut.moveMouse(Math.round(x), Math.round(y));
-                    device_libnut.mouseClick('left', true);
+                    this.inputDriver.moveMouse(Math.round(x), Math.round(y));
+                    this.inputDriver.mouseClick('left', true);
                 },
                 rightClick: async ({ x, y })=>{
-                    node_assert(device_libnut, 'libnut not initialized');
-                    device_libnut.moveMouse(Math.round(x), Math.round(y));
-                    device_libnut.mouseClick('right');
+                    this.inputDriver.moveMouse(Math.round(x), Math.round(y));
+                    this.inputDriver.mouseClick('right');
                 },
                 hover: async ({ x, y })=>{
-                    node_assert(device_libnut, 'libnut not initialized');
-                    await smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
-                    await sleep(MOUSE_MOVE_EFFECT_WAIT);
+                    await this.inputDriver.smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
+                    await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
                 },
                 dragAndDrop: async (from, to)=>{
-                    node_assert(device_libnut, 'libnut not initialized');
-                    device_libnut.moveMouse(Math.round(from.x), Math.round(from.y));
-                    device_libnut.mouseToggle('down', 'left');
-                    await sleep(100);
-                    device_libnut.moveMouse(Math.round(to.x), Math.round(to.y));
-                    await sleep(100);
-                    device_libnut.mouseToggle('up', 'left');
+                    this.inputDriver.moveMouse(Math.round(from.x), Math.round(from.y));
+                    await this.inputDriver.withMouseButton('left', async ()=>{
+                        await this.inputDriver.delay(100);
+                        this.inputDriver.moveMouse(Math.round(to.x), Math.round(to.y));
+                        await this.inputDriver.delay(100);
+                    });
                 }
             },
             keyboard: {
                 typeText: async (value, opts)=>{
-                    node_assert(device_libnut, 'libnut not initialized');
                     const element = opts?.target;
                     if (element) {
                         const [x, y] = element.center;
-                        device_libnut.moveMouse(Math.round(x), Math.round(y));
-                        device_libnut.mouseClick('left');
-                        await sleep(INPUT_FOCUS_DELAY);
+                        this.inputDriver.moveMouse(Math.round(x), Math.round(y));
+                        this.inputDriver.mouseClick('left');
+                        await this.inputDriver.delay(INPUT_FOCUS_DELAY);
                         if (opts?.replace !== false) {
                             await this.selectAllAndDelete();
-                            await sleep(INPUT_CLEAR_DELAY);
+                            await this.inputDriver.delay(INPUT_CLEAR_DELAY);
                         }
                     }
                     await this.smartTypeString(value);
                 },
                 keyboardPress: async (keyName, opts)=>{
-                    node_assert(device_libnut, 'libnut not initialized');
                     const target = opts?.target;
                     if (target) {
                         const [x, y] = target.center;
-                        device_libnut.moveMouse(Math.round(x), Math.round(y));
-                        device_libnut.mouseClick('left');
-                        await sleep(50);
+                        this.inputDriver.moveMouse(Math.round(x), Math.round(y));
+                        this.inputDriver.mouseClick('left');
+                        await this.inputDriver.delay(50);
                     }
                     await this.pressKeyboardShortcut(keyName);
                 },
                 clearInput: async (target)=>{
-                    node_assert(device_libnut, 'libnut not initialized');
                     if (target) {
                         const element = target;
                         const [x, y] = element.center;
-                        device_libnut.moveMouse(Math.round(x), Math.round(y));
-                        device_libnut.mouseClick('left');
-                        await sleep(100);
+                        this.inputDriver.moveMouse(Math.round(x), Math.round(y));
+                        this.inputDriver.mouseClick('left');
+                        await this.inputDriver.delay(100);
                     }
                     await this.selectAllAndDelete();
-                    await sleep(50);
+                    await this.inputDriver.delay(50);
                 }
             },
             scroll: {
@@ -1131,7 +1240,7 @@ class HelperProcessRDPBackendClient {
 function createDefaultRDPBackendClient() {
     return new HelperProcessRDPBackendClient();
 }
-function device_define_property(obj, key, value) {
+function rdp_device_define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
         enumerable: true,
@@ -1290,14 +1399,14 @@ class RDPDevice {
         }
     }
     constructor(options){
-        device_define_property(this, "interfaceType", 'rdp');
-        device_define_property(this, "options", void 0);
-        device_define_property(this, "backend", void 0);
-        device_define_property(this, "connectionInfo", void 0);
-        device_define_property(this, "destroyed", false);
-        device_define_property(this, "cursorPosition", void 0);
-        device_define_property(this, "uri", void 0);
-        device_define_property(this, "inputPrimitives", {
+        rdp_device_define_property(this, "interfaceType", 'rdp');
+        rdp_device_define_property(this, "options", void 0);
+        rdp_device_define_property(this, "backend", void 0);
+        rdp_device_define_property(this, "connectionInfo", void 0);
+        rdp_device_define_property(this, "destroyed", false);
+        rdp_device_define_property(this, "cursorPosition", void 0);
+        rdp_device_define_property(this, "uri", void 0);
+        rdp_device_define_property(this, "inputPrimitives", {
             pointer: {
                 tap: async ({ x, y })=>{
                     await this.movePointer(Math.round(x), Math.round(y), {
@@ -1629,7 +1738,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
     }
 }
 function version() {
-    const currentVersion = "1.8.6";
+    const currentVersion = "1.8.7-beta-20260527113633.0";
     console.log(`@midscene/computer v${currentVersion}`);
     return currentVersion;
 }