@midscene/computer 1.12.7-beta-20260911063016.0 → 1.12.7

package/README.md

@@ -1,53 +1,5 @@
 # @midscene/computer
 
-Midscene.js Computer Desktop Automation - AI-powered desktop automation for:
-
-- local desktop control on Windows, macOS, and Linux
-- remote Windows desktop control over the RDP protocol
+Desktop automation library for Midscene, providing AI-powered testing and automation capabilities for Windows, macOS, and Linux, with RDP support for remote Windows desktops.
 
 See <https://midscenejs.com/platforms/desktop>.
-
-## RDP support
-
-Use `agentForRDPComputer()`:
-
-```ts
-import { agentForRDPComputer } from '@midscene/computer';
-
-const agent = await agentForRDPComputer({
-  host: '10.0.0.10',
-  username: 'Admin',
-  password: 'secret',
-  ignoreCertificate: true,
-});
-```
-
-When the machine running Midscene has multiple outbound routes, pass
-`localAddress` to bind the RDP TCP connection to a specific local source IP:
-
-```ts
-const agent = await agentForRDPComputer({
-  host: '10.0.0.10',
-  username: 'Admin',
-  password: 'secret',
-  localAddress: '10.0.0.20',
-  ignoreCertificate: true,
-});
-```
-
-RDP usage requires:
-
-- a reachable Windows machine with RDP enabled
-- [FreeRDP](https://www.freerdp.com/) installed on the machine running your script
-
-If you need to rebuild the native helper locally from source:
-
-```bash
-pnpm --filter @midscene/computer run build:native
-```
-
-Run RDP AI tests:
-
-```bash
-pnpm --filter @midscene/computer run test:ai:rdp
-```

package/dist/es/cli.mjs

@@ -19,6 +19,73 @@ import node_assert from "node:assert";
 import { hasActiveCliInterruptWaiter } from "@midscene/shared/cli/interrupt";
 import { once } from "node:events";
 import { createInterface } from "node:readline";
+const debugKeyboard = getDebug('computer:keyboard');
+const APPLE_SCRIPT_KEY_CODES = {
+    return: 36,
+    enter: 36,
+    tab: 48,
+    space: 49,
+    backspace: 51,
+    delete: 51,
+    escape: 53,
+    forwarddelete: 117,
+    left: 123,
+    right: 124,
+    down: 125,
+    up: 126,
+    home: 115,
+    end: 119,
+    pageup: 116,
+    pagedown: 121,
+    f1: 122,
+    f2: 120,
+    f3: 99,
+    f4: 118,
+    f5: 96,
+    f6: 97,
+    f7: 98,
+    f8: 100,
+    f9: 101,
+    f10: 109,
+    f11: 103,
+    f12: 111
+};
+const APPLE_SCRIPT_MODIFIER_KEYS = {
+    command: 'command',
+    cmd: 'command',
+    control: 'control',
+    ctrl: 'control',
+    shift: 'shift',
+    alt: 'option',
+    option: 'option',
+    meta: 'command'
+};
+function buildKeyCommand(key) {
+    const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
+    if (void 0 !== keyCode) return `key code ${keyCode}`;
+    const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+    return `keystroke "${escapedKey}"`;
+}
+function resolveModifierKeys(modifiers) {
+    return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
+}
+function buildAppleScriptKeyPress(key, modifiers = []) {
+    const modifierKeys = resolveModifierKeys(modifiers);
+    const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
+    return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
+}
+function sendKeyViaAppleScript(key, modifiers = []) {
+    const script = buildAppleScriptKeyPress(key, modifiers);
+    debugKeyboard('sendKeyViaAppleScript', {
+        key,
+        modifiers,
+        script
+    });
+    execFileSync("osascript", [
+        '-e',
+        script
+    ]);
+}
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -91,6 +158,41 @@ class ComputerInputDriver {
         if (void 0 !== modifiers) lib.keyTap(key, modifiers);
         else lib.keyTap(key);
     }
+    keyToggle(key, state, modifiers) {
+        const lib = this.getLibnutOrThrow('keyToggle');
+        if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
+        else lib.keyToggle(key, state);
+    }
+    async keyTapWithModifierDelay(key, modifiers, delayMs) {
+        const uniqueModifiers = [
+            ...new Set(modifiers)
+        ];
+        if (0 === uniqueModifiers.length) return void this.keyTap(key);
+        if (0 === delayMs) return void this.keyTap(key, uniqueModifiers);
+        const pressedModifiers = [];
+        let failure;
+        try {
+            for (const modifier of uniqueModifiers){
+                this.keyToggle(modifier, 'down');
+                pressedModifiers.push(modifier);
+                await this.delay(delayMs);
+            }
+            this.keyTap(key);
+            await this.delay(delayMs);
+        } catch (error) {
+            failure = error;
+        }
+        for (const modifier of pressedModifiers.reverse()){
+            const releaseFailure = this.tryReleaseKey(modifier);
+            if (void 0 === failure && void 0 !== releaseFailure) failure = releaseFailure;
+            if (void 0 === failure) try {
+                await this.delay(delayMs);
+            } catch (error) {
+                failure = error;
+            }
+        }
+        if (void 0 !== failure) throw failure;
+    }
     typeString(text) {
         this.getLibnutOrThrow('typeString').typeString(text);
     }
@@ -163,6 +265,20 @@ class ComputerInputDriver {
             this.options.debug(`Failed to release mouse button ${button}: ${error}`);
         }
     }
+    tryReleaseKey(key) {
+        try {
+            const libnut = this.options.getLibnut();
+            node_assert(libnut, 'libnut not initialized');
+            libnut.keyToggle(key, 'up');
+            return;
+        } catch (error) {
+            const releaseError = new Error(`Failed to release modifier key "${key}"`, {
+                cause: error
+            });
+            this.options.debug(`${releaseError.message}: ${error}`);
+            return releaseError;
+        }
+    }
     rejectPendingInputDelays() {
         const error = this.createDestroyedError('in-flight input');
         for (const waitRef of this.pendingInputDelayWaits){
@@ -180,6 +296,96 @@ class ComputerInputDriver {
         this.pendingInputDelayWaits = new Set();
     }
 }
+const US_SHIFTED_CHARACTER_KEYS = new Map([
+    [
+        '~',
+        '`'
+    ],
+    [
+        '!',
+        '1'
+    ],
+    [
+        '@',
+        '2'
+    ],
+    [
+        '#',
+        '3'
+    ],
+    [
+        '$',
+        '4'
+    ],
+    [
+        '%',
+        '5'
+    ],
+    [
+        '^',
+        '6'
+    ],
+    [
+        '&',
+        '7'
+    ],
+    [
+        '*',
+        '8'
+    ],
+    [
+        '(',
+        '9'
+    ],
+    [
+        ')',
+        '0'
+    ],
+    [
+        '_',
+        '-'
+    ],
+    [
+        '+',
+        '='
+    ],
+    [
+        '{',
+        '['
+    ],
+    [
+        '}',
+        ']'
+    ],
+    [
+        '|',
+        '\\'
+    ],
+    [
+        ':',
+        ';'
+    ],
+    [
+        '"',
+        "'"
+    ],
+    [
+        '<',
+        ','
+    ],
+    [
+        '>',
+        '.'
+    ],
+    [
+        '?',
+        '/'
+    ]
+]);
+function resolveShiftedKey(character, keyboardLayout) {
+    if (/^[A-Z]$/.test(character)) return character.toLowerCase();
+    return 'en-US' === keyboardLayout ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+}
 const POWERSHELL_TIMEOUT_MS = 15000;
 const POWERSHELL_MAX_BUFFER = 67108864;
 const WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE = `
@@ -543,92 +749,6 @@ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
 const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
 const LIBNUT_FALLBACK_MAX_DETENTS = 200;
 const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
-const LINUX_SHIFTED_CHARACTER_KEYS = new Map([
-    [
-        '~',
-        '`'
-    ],
-    [
-        '!',
-        '1'
-    ],
-    [
-        '@',
-        '2'
-    ],
-    [
-        '#',
-        '3'
-    ],
-    [
-        '$',
-        '4'
-    ],
-    [
-        '%',
-        '5'
-    ],
-    [
-        '^',
-        '6'
-    ],
-    [
-        '&',
-        '7'
-    ],
-    [
-        '*',
-        '8'
-    ],
-    [
-        '(',
-        '9'
-    ],
-    [
-        ')',
-        '0'
-    ],
-    [
-        '_',
-        '-'
-    ],
-    [
-        '+',
-        '='
-    ],
-    [
-        '{',
-        '['
-    ],
-    [
-        '}',
-        ']'
-    ],
-    [
-        '|',
-        '\\'
-    ],
-    [
-        ':',
-        ';'
-    ],
-    [
-        '"',
-        "'"
-    ],
-    [
-        '<',
-        ','
-    ],
-    [
-        '>',
-        '.'
-    ],
-    [
-        '?',
-        '/'
-    ]
-]);
 const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
 const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
 const EDGE_SCROLL_SPEC = {
@@ -665,67 +785,6 @@ const EDGE_SCROLL_SPEC = {
         ]
     }
 };
-const APPLESCRIPT_KEY_CODE_MAP = {
-    return: 36,
-    enter: 36,
-    tab: 48,
-    space: 49,
-    backspace: 51,
-    delete: 51,
-    escape: 53,
-    forwarddelete: 117,
-    left: 123,
-    right: 124,
-    down: 125,
-    up: 126,
-    home: 115,
-    end: 119,
-    pageup: 116,
-    pagedown: 121,
-    f1: 122,
-    f2: 120,
-    f3: 99,
-    f4: 118,
-    f5: 96,
-    f6: 97,
-    f7: 98,
-    f8: 100,
-    f9: 101,
-    f10: 109,
-    f11: 103,
-    f12: 111
-};
-const APPLESCRIPT_MODIFIER_MAP = {
-    command: 'command down',
-    cmd: 'command down',
-    control: 'control down',
-    ctrl: 'control down',
-    shift: 'shift down',
-    alt: 'option down',
-    option: 'option down',
-    meta: 'command down'
-};
-function sendKeyViaAppleScript(key, modifiers = []) {
-    const lowerKey = key.toLowerCase();
-    const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
-    const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
-    const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
-    let script;
-    if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
-    else {
-        const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-        script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
-    }
-    debugDevice('sendKeyViaAppleScript', {
-        key,
-        modifiers,
-        script
-    });
-    execFileSync("osascript", [
-        '-e',
-        script
-    ]);
-}
 function escapePowershellSingleQuoted(value) {
     return value.replace(/'/g, "''");
 }
@@ -1153,7 +1212,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck(displays) {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.12.7-beta-20260911063016.0");
+        console.log("[HealthCheck] @midscene/computer v1.12.7");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -1330,9 +1389,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             ]);
             else {
                 const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                this.inputDriver.keyTap('v', [
+                await this.inputDriver.keyTapWithModifierDelay('v', [
                     modifier
-                ]);
+                ], this.options?.keyboardModifierDelay ?? 0);
             }
             await this.inputDriver.delay(100);
         } finally{
@@ -1347,12 +1406,16 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
     }
     async typeStringWithDelay(text, keyboardTypeDelay) {
         await sendTextSequentially(text.replace(/\r\n?/g, '\n'), {
-            sendCharacter: (character)=>{
-                const linuxShiftedKey = 'linux' === process.platform ? LINUX_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+            sendCharacter: async (character)=>{
+                const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+                const pacedShiftedKey = !this.useAppleScript && (this.options?.keyboardModifierDelay ?? 0) > 0 ? resolveShiftedKey(character, this.options?.keyboardLayout) : void 0;
                 if ('\n' === character) this.inputDriver.sendKey('return');
                 else if ('\t' === character) this.inputDriver.sendKey('tab');
                 else if (' ' === character) this.inputDriver.sendKey('space');
                 else if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript(character);
+                else if (void 0 !== pacedShiftedKey) await this.inputDriver.keyTapWithModifierDelay(pacedShiftedKey, [
+                    'shift'
+                ], this.options?.keyboardModifierDelay ?? 0);
                 else if (void 0 !== linuxShiftedKey) this.inputDriver.keyTap(linuxShiftedKey, [
                     'shift'
                 ]);
@@ -1373,9 +1436,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             return;
         }
         const modifier = 'darwin' === process.platform ? 'command' : 'control';
-        this.inputDriver.keyTap('a', [
+        await this.inputDriver.keyTapWithModifierDelay('a', [
             modifier
-        ]);
+        ], this.options?.keyboardModifierDelay ?? 0);
         await this.inputDriver.delay(50);
         this.inputDriver.keyTap('backspace');
     }
@@ -1389,6 +1452,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             modifiers,
             driver: this.useAppleScript ? "applescript" : 'libnut'
         });
+        if (!this.useAppleScript && modifiers.length > 0 && (this.options?.keyboardModifierDelay ?? 0) > 0) return void await this.inputDriver.keyTapWithModifierDelay(key, modifiers, this.options?.keyboardModifierDelay ?? 0);
         this.inputDriver.sendKey(key, modifiers);
     }
     resolveUntargetedScrollPoint(screenSize) {
@@ -1542,7 +1606,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
         device_define_property(this, "inputDriver", new ComputerInputDriver({
             getLibnut: ()=>device_libnut,
             useAppleScript: ()=>this.useAppleScript,
-            sendKeyViaAppleScript,
+            sendKeyViaAppleScript: sendKeyViaAppleScript,
             runPhasedScroll,
             debug: (message)=>debugDevice(message)
         }));
@@ -1668,6 +1732,8 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
                 }
             }
         });
+        if (options?.keyboardModifierDelay !== void 0 && (!Number.isFinite(options.keyboardModifierDelay) || options.keyboardModifierDelay < 0)) throw new Error('keyboardModifierDelay must be a finite non-negative number');
+        if (options?.keyboardLayout !== void 0 && 'en-US' !== options.keyboardLayout) throw new Error('keyboardLayout must be "en-US" when specified');
         this.options = options;
         this.displayId = options?.displayId;
         this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -2370,6 +2436,8 @@ function createLocalComputerDevice(opts) {
         keyboardTypeDelay: opts?.keyboardTypeDelay,
         inputStrategy: opts?.inputStrategy,
         keyboardDriver: opts?.keyboardDriver,
+        keyboardModifierDelay: opts?.keyboardModifierDelay,
+        keyboardLayout: opts?.keyboardLayout,
         headless: opts?.headless,
         xvfbResolution: opts?.xvfbResolution,
         keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
@@ -2427,6 +2495,10 @@ const computerInitArgShape = {
     headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
     keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
     inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
+    keyboardModifierDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds after each modifier transition and the main key for local libnut keyboard events. Positive values pace shortcuts and the implicit Shift used by uppercase letters during sequential input. Layout-dependent punctuation also requires keyboardLayout. Ignored in RDP mode and by the macOS AppleScript driver.'),
+    keyboardLayout: z["enum"]([
+        'en-US'
+    ]).optional().describe('Keyboard layout used to resolve layout-dependent shifted punctuation to base keys during sequential local libnut input. Currently only "en-US" is supported. Ignored in RDP mode and by the macOS AppleScript driver.'),
     host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
     port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
     username: z.string().optional().describe('RDP username. Requires host.'),
@@ -2443,7 +2515,7 @@ const computerInitArgShape = {
 function adaptComputerInitArgs(extracted) {
     if (!extracted || 0 === Object.keys(extracted).length) return;
     if (extracted.host) {
-        const { displayId: _d, headless: _h, ...rdpFields } = extracted;
+        const { displayId: _d, headless: _h, keyboardModifierDelay: _s, keyboardLayout: _l, ...rdpFields } = extracted;
         const host = normalizeRdpHost(extracted.host);
         return {
             mode: 'rdp',
@@ -2457,6 +2529,8 @@ function adaptComputerInitArgs(extracted) {
         headless: extracted.headless,
         keyboardTypeDelay: extracted.keyboardTypeDelay,
         inputStrategy: extracted.inputStrategy,
+        keyboardModifierDelay: extracted.keyboardModifierDelay,
+        keyboardLayout: extracted.keyboardLayout,
         ...extractAgentBehaviorInitArgs(extracted) ?? {}
     };
 }
@@ -2508,6 +2582,8 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
         const headless = opts?.mode === 'local' ? opts.headless : void 0;
         const keyboardTypeDelay = opts?.keyboardTypeDelay;
         const inputStrategy = opts?.inputStrategy;
+        const keyboardModifierDelay = opts?.mode === 'local' ? opts.keyboardModifierDelay : void 0;
+        const keyboardLayout = opts?.mode === 'local' ? opts.keyboardLayout : void 0;
         agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
         const agentOpts = {
             ...displayId ? {
@@ -2522,6 +2598,12 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
             ...void 0 !== inputStrategy ? {
                 inputStrategy
             } : {},
+            ...void 0 !== keyboardModifierDelay ? {
+                keyboardModifierDelay
+            } : {},
+            ...void 0 !== keyboardLayout ? {
+                keyboardLayout
+            } : {},
             ...this.options.keepXvfbAliveUntilProcessExit ? {
                 keepXvfbAliveUntilProcessExit: true
             } : {},
@@ -2606,7 +2688,7 @@ const tools = new ComputerMidsceneTools({
 });
 runToolsCLI(tools, 'midscene-computer', {
     stripPrefix: 'computer_',
-    version: "1.12.7-beta-20260911063016.0",
+    version: "1.12.7",
     extraCommands: createReportCliCommands()
 }).catch((e)=>{
     process.exit(reportCLIError(e));