@midscene/computer 1.12.4-beta-20260904041024.0 → 1.12.4

package/README.md

@@ -7,30 +7,6 @@ Midscene.js Computer Desktop Automation - AI-powered desktop automation for:
 
 See <https://midscenejs.com/platforms/desktop>.
 
-## VNC keyboard input on macOS
-
-When Midscene runs on macOS and controls a foreground VNC client, enable
-physical keyboard events so modifier keys are sent as explicit key-down and
-key-up transitions. Text must also use sequential input; a positive
-`keyboardTypeDelay` enables that behavior in the default `legacy` input mode:
-
-```ts
-import { agentForComputer } from '@midscene/computer';
-
-const agent = await agentForComputer({
-  keyboardEventMode: 'physical',
-  keyboardTypeDelay: 80,
-});
-```
-
-The default `keyboardEventMode: 'logical'` keeps the standard AppleScript
-behavior for non-VNC applications. This option is ignored outside macOS and
-when `keyboardDriver` is set to `libnut`.
-
-Use `physical` only for a VNC client with matching en-US keyboard layouts. Its
-shifted-punctuation mapping is not layout-independent, and native macOS apps
-may interpret the base key directly—for example, `!@#` can become `123`.
-
 ## RDP support
 
 Use `agentForRDPComputer()`:

package/dist/es/cli.mjs

@@ -19,210 +19,6 @@ import node_assert from "node:assert";
 import { hasActiveCliInterruptWaiter } from "@midscene/shared/cli/interrupt";
 import { once } from "node:events";
 import { createInterface } from "node:readline";
-const US_SHIFTED_CHARACTER_KEYS = new Map([
-    [
-        '~',
-        '`'
-    ],
-    [
-        '!',
-        '1'
-    ],
-    [
-        '@',
-        '2'
-    ],
-    [
-        '#',
-        '3'
-    ],
-    [
-        '$',
-        '4'
-    ],
-    [
-        '%',
-        '5'
-    ],
-    [
-        '^',
-        '6'
-    ],
-    [
-        '&',
-        '7'
-    ],
-    [
-        '*',
-        '8'
-    ],
-    [
-        '(',
-        '9'
-    ],
-    [
-        ')',
-        '0'
-    ],
-    [
-        '_',
-        '-'
-    ],
-    [
-        '+',
-        '='
-    ],
-    [
-        '{',
-        '['
-    ],
-    [
-        '}',
-        ']'
-    ],
-    [
-        '|',
-        '\\'
-    ],
-    [
-        ':',
-        ';'
-    ],
-    [
-        '"',
-        "'"
-    ],
-    [
-        '<',
-        ','
-    ],
-    [
-        '>',
-        '.'
-    ],
-    [
-        '?',
-        '/'
-    ]
-]);
-const debugKeyboard = getDebug('computer:keyboard');
-const APPLE_SCRIPT_KEY_CODES = {
-    return: 36,
-    enter: 36,
-    tab: 48,
-    space: 49,
-    backspace: 51,
-    delete: 51,
-    escape: 53,
-    forwarddelete: 117,
-    left: 123,
-    right: 124,
-    down: 125,
-    up: 126,
-    home: 115,
-    end: 119,
-    pageup: 116,
-    pagedown: 121,
-    f1: 122,
-    f2: 120,
-    f3: 99,
-    f4: 118,
-    f5: 96,
-    f6: 97,
-    f7: 98,
-    f8: 100,
-    f9: 101,
-    f10: 109,
-    f11: 103,
-    f12: 111
-};
-const APPLE_SCRIPT_MODIFIER_KEYS = {
-    command: 'command',
-    cmd: 'command',
-    control: 'control',
-    ctrl: 'control',
-    shift: 'shift',
-    alt: 'option',
-    option: 'option',
-    meta: 'command'
-};
-function buildKeyCommand(key) {
-    const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
-    if (void 0 !== keyCode) return `key code ${keyCode}`;
-    const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-    return `keystroke "${escapedKey}"`;
-}
-function resolveModifierKeys(modifiers) {
-    return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
-}
-function buildLogicalKeyPress(key, modifiers) {
-    const modifierKeys = resolveModifierKeys(modifiers);
-    const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
-    return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
-}
-function resolvePhysicalKey(key, modifiers) {
-    const resolvedModifiers = [
-        ...modifiers
-    ];
-    const shiftedBaseKey = US_SHIFTED_CHARACTER_KEYS.get(key);
-    if (/^[A-Z]$/.test(key)) {
-        resolvedModifiers.push('shift');
-        return {
-            key: key.toLowerCase(),
-            modifiers: resolvedModifiers
-        };
-    }
-    if (void 0 !== shiftedBaseKey) {
-        resolvedModifiers.push('shift');
-        return {
-            key: shiftedBaseKey,
-            modifiers: resolvedModifiers
-        };
-    }
-    return {
-        key,
-        modifiers: resolvedModifiers
-    };
-}
-function buildPhysicalKeyPress(key, modifiers) {
-    const resolved = resolvePhysicalKey(key, modifiers);
-    const modifierKeys = [
-        ...new Set(resolveModifierKeys(resolved.modifiers))
-    ];
-    const keyCommand = buildKeyCommand(resolved.key);
-    if (0 === modifierKeys.length) return `tell application "System Events" to ${keyCommand}`;
-    const releaseCommands = [
-        ...modifierKeys
-    ].reverse().map((modifier)=>`key up ${modifier}`);
-    return [
-        'tell application "System Events"',
-        'try',
-        ...modifierKeys.map((modifier)=>`key down ${modifier}`),
-        keyCommand,
-        'on error errorMessage number errorNumber',
-        ...releaseCommands,
-        'error errorMessage number errorNumber',
-        'end try',
-        ...releaseCommands,
-        'end tell'
-    ].join('\n');
-}
-function buildAppleScriptKeyPress(key, modifiers = [], eventMode = 'logical') {
-    return 'physical' === eventMode ? buildPhysicalKeyPress(key, modifiers) : buildLogicalKeyPress(key, modifiers);
-}
-function sendKeyViaAppleScript(key, modifiers = [], eventMode = 'logical') {
-    const script = buildAppleScriptKeyPress(key, modifiers, eventMode);
-    debugKeyboard('sendKeyViaAppleScript', {
-        key,
-        modifiers,
-        eventMode,
-        script
-    });
-    execFileSync("osascript", [
-        '-e',
-        script
-    ]);
-}
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -745,6 +541,92 @@ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
 const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
 const LIBNUT_FALLBACK_MAX_DETENTS = 200;
 const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
+const LINUX_SHIFTED_CHARACTER_KEYS = new Map([
+    [
+        '~',
+        '`'
+    ],
+    [
+        '!',
+        '1'
+    ],
+    [
+        '@',
+        '2'
+    ],
+    [
+        '#',
+        '3'
+    ],
+    [
+        '$',
+        '4'
+    ],
+    [
+        '%',
+        '5'
+    ],
+    [
+        '^',
+        '6'
+    ],
+    [
+        '&',
+        '7'
+    ],
+    [
+        '*',
+        '8'
+    ],
+    [
+        '(',
+        '9'
+    ],
+    [
+        ')',
+        '0'
+    ],
+    [
+        '_',
+        '-'
+    ],
+    [
+        '+',
+        '='
+    ],
+    [
+        '{',
+        '['
+    ],
+    [
+        '}',
+        ']'
+    ],
+    [
+        '|',
+        '\\'
+    ],
+    [
+        ':',
+        ';'
+    ],
+    [
+        '"',
+        "'"
+    ],
+    [
+        '<',
+        ','
+    ],
+    [
+        '>',
+        '.'
+    ],
+    [
+        '?',
+        '/'
+    ]
+]);
 const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
 const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
 const EDGE_SCROLL_SPEC = {
@@ -781,6 +663,67 @@ const EDGE_SCROLL_SPEC = {
         ]
     }
 };
+const APPLESCRIPT_KEY_CODE_MAP = {
+    return: 36,
+    enter: 36,
+    tab: 48,
+    space: 49,
+    backspace: 51,
+    delete: 51,
+    escape: 53,
+    forwarddelete: 117,
+    left: 123,
+    right: 124,
+    down: 125,
+    up: 126,
+    home: 115,
+    end: 119,
+    pageup: 116,
+    pagedown: 121,
+    f1: 122,
+    f2: 120,
+    f3: 99,
+    f4: 118,
+    f5: 96,
+    f6: 97,
+    f7: 98,
+    f8: 100,
+    f9: 101,
+    f10: 109,
+    f11: 103,
+    f12: 111
+};
+const APPLESCRIPT_MODIFIER_MAP = {
+    command: 'command down',
+    cmd: 'command down',
+    control: 'control down',
+    ctrl: 'control down',
+    shift: 'shift down',
+    alt: 'option down',
+    option: 'option down',
+    meta: 'command down'
+};
+function sendKeyViaAppleScript(key, modifiers = []) {
+    const lowerKey = key.toLowerCase();
+    const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
+    const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
+    const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
+    let script;
+    if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
+    else {
+        const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+        script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
+    }
+    debugDevice('sendKeyViaAppleScript', {
+        key,
+        modifiers,
+        script
+    });
+    execFileSync("osascript", [
+        '-e',
+        script
+    ]);
+}
 function escapePowershellSingleQuoted(value) {
     return value.replace(/'/g, "''");
 }
@@ -1208,7 +1151,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck(displays) {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.12.4-beta-20260904041024.0");
+        console.log("[HealthCheck] @midscene/computer v1.12.4");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -1403,7 +1346,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
     async typeStringWithDelay(text, keyboardTypeDelay) {
         await sendTextSequentially(text.replace(/\r\n?/g, '\n'), {
             sendCharacter: (character)=>{
-                const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+                const linuxShiftedKey = 'linux' === process.platform ? LINUX_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
                 if ('\n' === character) this.inputDriver.sendKey('return');
                 else if ('\t' === character) this.inputDriver.sendKey('tab');
                 else if (' ' === character) this.inputDriver.sendKey('space');
@@ -1597,7 +1540,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
         device_define_property(this, "inputDriver", new ComputerInputDriver({
             getLibnut: ()=>device_libnut,
             useAppleScript: ()=>this.useAppleScript,
-            sendKeyViaAppleScript: (key, modifiers)=>sendKeyViaAppleScript(key, modifiers, this.options?.keyboardEventMode ?? 'logical'),
+            sendKeyViaAppleScript,
             runPhasedScroll,
             debug: (message)=>debugDevice(message)
         }));
@@ -2422,7 +2365,6 @@ function createLocalComputerDevice(opts) {
         keyboardTypeDelay: opts?.keyboardTypeDelay,
         inputStrategy: opts?.inputStrategy,
         keyboardDriver: opts?.keyboardDriver,
-        keyboardEventMode: opts?.keyboardEventMode,
         headless: opts?.headless,
         xvfbResolution: opts?.xvfbResolution,
         keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
@@ -2480,10 +2422,6 @@ const computerInitArgShape = {
     headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
     keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
     inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
-    keyboardEventMode: z["enum"]([
-        'logical',
-        'physical'
-    ]).optional().describe('macOS AppleScript keyboard event mode for local control. "logical" (default) targets native apps; "physical" is only for VNC clients, assumes an en-US layout for shifted punctuation, and requires sequential text input or a positive keyboardTypeDelay. Ignored in RDP mode and outside the macOS AppleScript driver.'),
     host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
     port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
     username: z.string().optional().describe('RDP username. Requires host.'),
@@ -2500,7 +2438,7 @@ const computerInitArgShape = {
 function adaptComputerInitArgs(extracted) {
     if (!extracted || 0 === Object.keys(extracted).length) return;
     if (extracted.host) {
-        const { displayId: _d, headless: _h, keyboardEventMode: _k, ...rdpFields } = extracted;
+        const { displayId: _d, headless: _h, ...rdpFields } = extracted;
         const host = normalizeRdpHost(extracted.host);
         return {
             mode: 'rdp',
@@ -2514,7 +2452,6 @@ function adaptComputerInitArgs(extracted) {
         headless: extracted.headless,
         keyboardTypeDelay: extracted.keyboardTypeDelay,
         inputStrategy: extracted.inputStrategy,
-        keyboardEventMode: extracted.keyboardEventMode,
         ...extractAgentBehaviorInitArgs(extracted) ?? {}
     };
 }
@@ -2566,7 +2503,6 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
         const headless = opts?.mode === 'local' ? opts.headless : void 0;
         const keyboardTypeDelay = opts?.keyboardTypeDelay;
         const inputStrategy = opts?.inputStrategy;
-        const keyboardEventMode = opts?.mode === 'local' ? opts.keyboardEventMode : void 0;
         agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
         const agentOpts = {
             ...displayId ? {
@@ -2581,9 +2517,6 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
             ...void 0 !== inputStrategy ? {
                 inputStrategy
             } : {},
-            ...void 0 !== keyboardEventMode ? {
-                keyboardEventMode
-            } : {},
             ...this.options.keepXvfbAliveUntilProcessExit ? {
                 keepXvfbAliveUntilProcessExit: true
             } : {},
@@ -2668,7 +2601,7 @@ const tools = new ComputerMidsceneTools({
 });
 runToolsCLI(tools, 'midscene-computer', {
     stripPrefix: 'computer_',
-    version: "1.12.4-beta-20260904041024.0",
+    version: "1.12.4",
     extraCommands: createReportCliCommands()
 }).catch((e)=>{
     process.exit(reportCLIError(e));