@midscene/computer 1.12.5 → 1.12.6-beta-20260909033030.0

package/README.md

@@ -1,53 +1,5 @@
 # @midscene/computer
 
-Midscene.js Computer Desktop Automation - AI-powered desktop automation for:
-
-- local desktop control on Windows, macOS, and Linux
-- remote Windows desktop control over the RDP protocol
+Desktop automation library for Midscene, providing AI-powered testing and automation capabilities for Windows, macOS, and Linux, with RDP support for remote Windows desktops.
 
 See <https://midscenejs.com/platforms/desktop>.
-
-## RDP support
-
-Use `agentForRDPComputer()`:
-
-```ts
-import { agentForRDPComputer } from '@midscene/computer';
-
-const agent = await agentForRDPComputer({
-  host: '10.0.0.10',
-  username: 'Admin',
-  password: 'secret',
-  ignoreCertificate: true,
-});
-```
-
-When the machine running Midscene has multiple outbound routes, pass
-`localAddress` to bind the RDP TCP connection to a specific local source IP:
-
-```ts
-const agent = await agentForRDPComputer({
-  host: '10.0.0.10',
-  username: 'Admin',
-  password: 'secret',
-  localAddress: '10.0.0.20',
-  ignoreCertificate: true,
-});
-```
-
-RDP usage requires:
-
-- a reachable Windows machine with RDP enabled
-- [FreeRDP](https://www.freerdp.com/) installed on the machine running your script
-
-If you need to rebuild the native helper locally from source:
-
-```bash
-pnpm --filter @midscene/computer run build:native
-```
-
-Run RDP AI tests:
-
-```bash
-pnpm --filter @midscene/computer run test:ai:rdp
-```

package/dist/es/cli.mjs

@@ -19,6 +19,207 @@ import node_assert from "node:assert";
 import { hasActiveCliInterruptWaiter } from "@midscene/shared/cli/interrupt";
 import { once } from "node:events";
 import { createInterface } from "node:readline";
+const US_SHIFTED_CHARACTER_KEYS = new Map([
+    [
+        '~',
+        '`'
+    ],
+    [
+        '!',
+        '1'
+    ],
+    [
+        '@',
+        '2'
+    ],
+    [
+        '#',
+        '3'
+    ],
+    [
+        '$',
+        '4'
+    ],
+    [
+        '%',
+        '5'
+    ],
+    [
+        '^',
+        '6'
+    ],
+    [
+        '&',
+        '7'
+    ],
+    [
+        '*',
+        '8'
+    ],
+    [
+        '(',
+        '9'
+    ],
+    [
+        ')',
+        '0'
+    ],
+    [
+        '_',
+        '-'
+    ],
+    [
+        '+',
+        '='
+    ],
+    [
+        '{',
+        '['
+    ],
+    [
+        '}',
+        ']'
+    ],
+    [
+        '|',
+        '\\'
+    ],
+    [
+        ':',
+        ';'
+    ],
+    [
+        '"',
+        "'"
+    ],
+    [
+        '<',
+        ','
+    ],
+    [
+        '>',
+        '.'
+    ],
+    [
+        '?',
+        '/'
+    ]
+]);
+function resolveUSShiftedKey(character) {
+    if (/^[A-Z]$/.test(character)) return character.toLowerCase();
+    return US_SHIFTED_CHARACTER_KEYS.get(character);
+}
+const debugKeyboard = getDebug('computer:keyboard');
+const APPLE_SCRIPT_KEY_CODES = {
+    return: 36,
+    enter: 36,
+    tab: 48,
+    space: 49,
+    backspace: 51,
+    delete: 51,
+    escape: 53,
+    forwarddelete: 117,
+    left: 123,
+    right: 124,
+    down: 125,
+    up: 126,
+    home: 115,
+    end: 119,
+    pageup: 116,
+    pagedown: 121,
+    f1: 122,
+    f2: 120,
+    f3: 99,
+    f4: 118,
+    f5: 96,
+    f6: 97,
+    f7: 98,
+    f8: 100,
+    f9: 101,
+    f10: 109,
+    f11: 103,
+    f12: 111
+};
+const APPLE_SCRIPT_MODIFIER_KEYS = {
+    command: 'command',
+    cmd: 'command',
+    control: 'control',
+    ctrl: 'control',
+    shift: 'shift',
+    alt: 'option',
+    option: 'option',
+    meta: 'command'
+};
+function buildKeyCommand(key) {
+    const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
+    if (void 0 !== keyCode) return `key code ${keyCode}`;
+    const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+    return `keystroke "${escapedKey}"`;
+}
+function resolveModifierKeys(modifiers) {
+    return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
+}
+function buildLogicalKeyPress(key, modifiers) {
+    const modifierKeys = resolveModifierKeys(modifiers);
+    const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
+    return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
+}
+function resolvePhysicalKey(key, modifiers) {
+    const resolvedModifiers = [
+        ...modifiers
+    ];
+    const shiftedBaseKey = resolveUSShiftedKey(key);
+    if (void 0 !== shiftedBaseKey) {
+        resolvedModifiers.push('shift');
+        return {
+            key: shiftedBaseKey,
+            modifiers: resolvedModifiers
+        };
+    }
+    return {
+        key,
+        modifiers: resolvedModifiers
+    };
+}
+function buildPhysicalKeyPress(key, modifiers) {
+    const resolved = resolvePhysicalKey(key, modifiers);
+    const modifierKeys = [
+        ...new Set(resolveModifierKeys(resolved.modifiers))
+    ];
+    const keyCommand = buildKeyCommand(resolved.key);
+    if (0 === modifierKeys.length) return `tell application "System Events" to ${keyCommand}`;
+    const releaseCommands = [
+        ...modifierKeys
+    ].reverse().map((modifier)=>`key up ${modifier}`);
+    return [
+        'tell application "System Events"',
+        'try',
+        ...modifierKeys.map((modifier)=>`key down ${modifier}`),
+        keyCommand,
+        'on error errorMessage number errorNumber',
+        ...releaseCommands,
+        'error errorMessage number errorNumber',
+        'end try',
+        ...releaseCommands,
+        'end tell'
+    ].join('\n');
+}
+function buildAppleScriptKeyPress(key, modifiers = [], eventMode = 'logical') {
+    return 'physical' === eventMode ? buildPhysicalKeyPress(key, modifiers) : buildLogicalKeyPress(key, modifiers);
+}
+function sendKeyViaAppleScript(key, modifiers = [], eventMode = 'logical') {
+    const script = buildAppleScriptKeyPress(key, modifiers, eventMode);
+    debugKeyboard('sendKeyViaAppleScript', {
+        key,
+        modifiers,
+        eventMode,
+        script
+    });
+    execFileSync("osascript", [
+        '-e',
+        script
+    ]);
+}
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -91,6 +292,29 @@ class ComputerInputDriver {
         if (void 0 !== modifiers) lib.keyTap(key, modifiers);
         else lib.keyTap(key);
     }
+    keyToggle(key, state, modifiers) {
+        const lib = this.getLibnutOrThrow('keyToggle');
+        if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
+        else lib.keyToggle(key, state);
+    }
+    async keyTapWithExplicitModifiers(key, modifiers, delayMs) {
+        const uniqueModifiers = [
+            ...new Set(modifiers)
+        ];
+        if (0 === uniqueModifiers.length) return void this.keyTap(key);
+        const pressedModifiers = [];
+        try {
+            for (const modifier of uniqueModifiers){
+                this.keyToggle(modifier, 'down');
+                pressedModifiers.push(modifier);
+            }
+            await this.delay(delayMs);
+            this.keyTap(key);
+            await this.delay(delayMs);
+        } finally{
+            for (const modifier of pressedModifiers.reverse())this.releaseKey(modifier);
+        }
+    }
     typeString(text) {
         this.getLibnutOrThrow('typeString').typeString(text);
     }
@@ -163,6 +387,15 @@ class ComputerInputDriver {
             this.options.debug(`Failed to release mouse button ${button}: ${error}`);
         }
     }
+    releaseKey(key) {
+        try {
+            const libnut = this.options.getLibnut();
+            node_assert(libnut, 'libnut not initialized');
+            libnut.keyToggle(key, 'up');
+        } catch (error) {
+            this.options.debug(`Failed to release key ${key}: ${error}`);
+        }
+    }
     rejectPendingInputDelays() {
         const error = this.createDestroyedError('in-flight input');
         for (const waitRef of this.pendingInputDelayWaits){
@@ -543,92 +776,6 @@ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
 const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
 const LIBNUT_FALLBACK_MAX_DETENTS = 200;
 const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
-const LINUX_SHIFTED_CHARACTER_KEYS = new Map([
-    [
-        '~',
-        '`'
-    ],
-    [
-        '!',
-        '1'
-    ],
-    [
-        '@',
-        '2'
-    ],
-    [
-        '#',
-        '3'
-    ],
-    [
-        '$',
-        '4'
-    ],
-    [
-        '%',
-        '5'
-    ],
-    [
-        '^',
-        '6'
-    ],
-    [
-        '&',
-        '7'
-    ],
-    [
-        '*',
-        '8'
-    ],
-    [
-        '(',
-        '9'
-    ],
-    [
-        ')',
-        '0'
-    ],
-    [
-        '_',
-        '-'
-    ],
-    [
-        '+',
-        '='
-    ],
-    [
-        '{',
-        '['
-    ],
-    [
-        '}',
-        ']'
-    ],
-    [
-        '|',
-        '\\'
-    ],
-    [
-        ':',
-        ';'
-    ],
-    [
-        '"',
-        "'"
-    ],
-    [
-        '<',
-        ','
-    ],
-    [
-        '>',
-        '.'
-    ],
-    [
-        '?',
-        '/'
-    ]
-]);
 const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
 const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
 const EDGE_SCROLL_SPEC = {
@@ -665,67 +812,6 @@ const EDGE_SCROLL_SPEC = {
         ]
     }
 };
-const APPLESCRIPT_KEY_CODE_MAP = {
-    return: 36,
-    enter: 36,
-    tab: 48,
-    space: 49,
-    backspace: 51,
-    delete: 51,
-    escape: 53,
-    forwarddelete: 117,
-    left: 123,
-    right: 124,
-    down: 125,
-    up: 126,
-    home: 115,
-    end: 119,
-    pageup: 116,
-    pagedown: 121,
-    f1: 122,
-    f2: 120,
-    f3: 99,
-    f4: 118,
-    f5: 96,
-    f6: 97,
-    f7: 98,
-    f8: 100,
-    f9: 101,
-    f10: 109,
-    f11: 103,
-    f12: 111
-};
-const APPLESCRIPT_MODIFIER_MAP = {
-    command: 'command down',
-    cmd: 'command down',
-    control: 'control down',
-    ctrl: 'control down',
-    shift: 'shift down',
-    alt: 'option down',
-    option: 'option down',
-    meta: 'command down'
-};
-function sendKeyViaAppleScript(key, modifiers = []) {
-    const lowerKey = key.toLowerCase();
-    const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
-    const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
-    const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
-    let script;
-    if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
-    else {
-        const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-        script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
-    }
-    debugDevice('sendKeyViaAppleScript', {
-        key,
-        modifiers,
-        script
-    });
-    execFileSync("osascript", [
-        '-e',
-        script
-    ]);
-}
 function escapePowershellSingleQuoted(value) {
     return value.replace(/'/g, "''");
 }
@@ -1153,7 +1239,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck(displays) {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.12.5");
+        console.log("[HealthCheck] @midscene/computer v1.12.6-beta-20260909033030.0");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -1330,7 +1416,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             ]);
             else {
                 const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                this.inputDriver.keyTap('v', [
+                await this.sendLocalModifiedKey('v', [
                     modifier
                 ]);
             }
@@ -1347,12 +1433,16 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
     }
     async typeStringWithDelay(text, keyboardTypeDelay) {
         await sendTextSequentially(text.replace(/\r\n?/g, '\n'), {
-            sendCharacter: (character)=>{
-                const linuxShiftedKey = 'linux' === process.platform ? LINUX_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+            sendCharacter: async (character)=>{
+                const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+                const pacedShiftedKey = !this.useAppleScript && (this.options?.keyboardShortcutDelay ?? 0) > 0 ? resolveUSShiftedKey(character) : void 0;
                 if ('\n' === character) this.inputDriver.sendKey('return');
                 else if ('\t' === character) this.inputDriver.sendKey('tab');
                 else if (' ' === character) this.inputDriver.sendKey('space');
                 else if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript(character);
+                else if (void 0 !== pacedShiftedKey) await this.sendLocalModifiedKey(pacedShiftedKey, [
+                    'shift'
+                ]);
                 else if (void 0 !== linuxShiftedKey) this.inputDriver.keyTap(linuxShiftedKey, [
                     'shift'
                 ]);
@@ -1373,12 +1463,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             return;
         }
         const modifier = 'darwin' === process.platform ? 'command' : 'control';
-        this.inputDriver.keyTap('a', [
+        await this.sendLocalModifiedKey('a', [
             modifier
         ]);
         await this.inputDriver.delay(50);
         this.inputDriver.keyTap('backspace');
     }
+    async sendLocalModifiedKey(key, modifiers) {
+        const shortcutDelay = this.options?.keyboardShortcutDelay ?? 0;
+        if (modifiers.length > 0 && shortcutDelay > 0) return void await this.inputDriver.keyTapWithExplicitModifiers(key, modifiers, shortcutDelay);
+        this.inputDriver.keyTap(key, modifiers);
+    }
     async pressKeyboardShortcut(keyName) {
         const keys = keyName.split('+');
         const modifiers = keys.slice(0, -1).map(normalizeKeyName);
@@ -1389,6 +1484,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             modifiers,
             driver: this.useAppleScript ? "applescript" : 'libnut'
         });
+        if (!this.useAppleScript && modifiers.length > 0 && (this.options?.keyboardShortcutDelay ?? 0) > 0) return void await this.sendLocalModifiedKey(key, modifiers);
         this.inputDriver.sendKey(key, modifiers);
     }
     resolveUntargetedScrollPoint(screenSize) {
@@ -1542,7 +1638,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
         device_define_property(this, "inputDriver", new ComputerInputDriver({
             getLibnut: ()=>device_libnut,
             useAppleScript: ()=>this.useAppleScript,
-            sendKeyViaAppleScript,
+            sendKeyViaAppleScript: (key, modifiers)=>sendKeyViaAppleScript(key, modifiers, this.options?.keyboardEventMode ?? 'logical'),
             runPhasedScroll,
             debug: (message)=>debugDevice(message)
         }));
@@ -1668,6 +1764,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
                 }
             }
         });
+        if (options?.keyboardShortcutDelay !== void 0 && (!Number.isFinite(options.keyboardShortcutDelay) || options.keyboardShortcutDelay < 0)) throw new Error('keyboardShortcutDelay must be a finite non-negative number');
         this.options = options;
         this.displayId = options?.displayId;
         this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -2370,6 +2467,8 @@ function createLocalComputerDevice(opts) {
         keyboardTypeDelay: opts?.keyboardTypeDelay,
         inputStrategy: opts?.inputStrategy,
         keyboardDriver: opts?.keyboardDriver,
+        keyboardEventMode: opts?.keyboardEventMode,
+        keyboardShortcutDelay: opts?.keyboardShortcutDelay,
         headless: opts?.headless,
         xvfbResolution: opts?.xvfbResolution,
         keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
@@ -2427,6 +2526,11 @@ const computerInitArgShape = {
     headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
     keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
     inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
+    keyboardEventMode: z["enum"]([
+        'logical',
+        'physical'
+    ]).optional().describe('macOS AppleScript keyboard event mode for local control. "logical" (default) uses a compact keystroke command; "physical" sends explicit modifier transitions for foreground apps that require separate key state changes. Both modes use AppleScript, and "physical" is not hardware input. It assumes an en-US layout for shifted punctuation and requires sequential text input or a positive keyboardTypeDelay. Ignored in RDP mode and outside the macOS AppleScript driver.'),
+    keyboardShortcutDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds around modifier transitions for local libnut keyboard events. Positive values pace shortcuts and the implicit Shift used by uppercase or en-US punctuation during sequential input. Ignored in RDP mode and by the macOS AppleScript driver.'),
     host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
     port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
     username: z.string().optional().describe('RDP username. Requires host.'),
@@ -2443,7 +2547,7 @@ const computerInitArgShape = {
 function adaptComputerInitArgs(extracted) {
     if (!extracted || 0 === Object.keys(extracted).length) return;
     if (extracted.host) {
-        const { displayId: _d, headless: _h, ...rdpFields } = extracted;
+        const { displayId: _d, headless: _h, keyboardEventMode: _k, keyboardShortcutDelay: _s, ...rdpFields } = extracted;
         const host = normalizeRdpHost(extracted.host);
         return {
             mode: 'rdp',
@@ -2457,6 +2561,8 @@ function adaptComputerInitArgs(extracted) {
         headless: extracted.headless,
         keyboardTypeDelay: extracted.keyboardTypeDelay,
         inputStrategy: extracted.inputStrategy,
+        keyboardEventMode: extracted.keyboardEventMode,
+        keyboardShortcutDelay: extracted.keyboardShortcutDelay,
         ...extractAgentBehaviorInitArgs(extracted) ?? {}
     };
 }
@@ -2508,6 +2614,8 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
         const headless = opts?.mode === 'local' ? opts.headless : void 0;
         const keyboardTypeDelay = opts?.keyboardTypeDelay;
         const inputStrategy = opts?.inputStrategy;
+        const keyboardEventMode = opts?.mode === 'local' ? opts.keyboardEventMode : void 0;
+        const keyboardShortcutDelay = opts?.mode === 'local' ? opts.keyboardShortcutDelay : void 0;
         agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
         const agentOpts = {
             ...displayId ? {
@@ -2522,6 +2630,12 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
             ...void 0 !== inputStrategy ? {
                 inputStrategy
             } : {},
+            ...void 0 !== keyboardEventMode ? {
+                keyboardEventMode
+            } : {},
+            ...void 0 !== keyboardShortcutDelay ? {
+                keyboardShortcutDelay
+            } : {},
             ...this.options.keepXvfbAliveUntilProcessExit ? {
                 keepXvfbAliveUntilProcessExit: true
             } : {},
@@ -2606,7 +2720,7 @@ const tools = new ComputerMidsceneTools({
 });
 runToolsCLI(tools, 'midscene-computer', {
     stripPrefix: 'computer_',
-    version: "1.12.5",
+    version: "1.12.6-beta-20260909033030.0",
     extraCommands: createReportCliCommands()
 }).catch((e)=>{
     process.exit(reportCLIError(e));