@midscene/computer 1.12.7-beta-20260911063016.0 → 1.12.7

package/dist/lib/index.js

@@ -68,6 +68,73 @@ const img_namespaceObject = require("@midscene/shared/img");
 const logger_namespaceObject = require("@midscene/shared/logger");
 const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
 var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
+const debugKeyboard = (0, logger_namespaceObject.getDebug)('computer:keyboard');
+const APPLE_SCRIPT_KEY_CODES = {
+    return: 36,
+    enter: 36,
+    tab: 48,
+    space: 49,
+    backspace: 51,
+    delete: 51,
+    escape: 53,
+    forwarddelete: 117,
+    left: 123,
+    right: 124,
+    down: 125,
+    up: 126,
+    home: 115,
+    end: 119,
+    pageup: 116,
+    pagedown: 121,
+    f1: 122,
+    f2: 120,
+    f3: 99,
+    f4: 118,
+    f5: 96,
+    f6: 97,
+    f7: 98,
+    f8: 100,
+    f9: 101,
+    f10: 109,
+    f11: 103,
+    f12: 111
+};
+const APPLE_SCRIPT_MODIFIER_KEYS = {
+    command: 'command',
+    cmd: 'command',
+    control: 'control',
+    ctrl: 'control',
+    shift: 'shift',
+    alt: 'option',
+    option: 'option',
+    meta: 'command'
+};
+function buildKeyCommand(key) {
+    const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
+    if (void 0 !== keyCode) return `key code ${keyCode}`;
+    const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+    return `keystroke "${escapedKey}"`;
+}
+function resolveModifierKeys(modifiers) {
+    return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
+}
+function buildAppleScriptKeyPress(key, modifiers = []) {
+    const modifierKeys = resolveModifierKeys(modifiers);
+    const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
+    return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
+}
+function sendKeyViaAppleScript(key, modifiers = []) {
+    const script = buildAppleScriptKeyPress(key, modifiers);
+    debugKeyboard('sendKeyViaAppleScript', {
+        key,
+        modifiers,
+        script
+    });
+    (0, external_node_child_process_namespaceObject.execFileSync)("osascript", [
+        '-e',
+        script
+    ]);
+}
 const external_node_assert_namespaceObject = require("node:assert");
 var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
 function _define_property(obj, key, value) {
@@ -142,6 +209,41 @@ class ComputerInputDriver {
         if (void 0 !== modifiers) lib.keyTap(key, modifiers);
         else lib.keyTap(key);
     }
+    keyToggle(key, state, modifiers) {
+        const lib = this.getLibnutOrThrow('keyToggle');
+        if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
+        else lib.keyToggle(key, state);
+    }
+    async keyTapWithModifierDelay(key, modifiers, delayMs) {
+        const uniqueModifiers = [
+            ...new Set(modifiers)
+        ];
+        if (0 === uniqueModifiers.length) return void this.keyTap(key);
+        if (0 === delayMs) return void this.keyTap(key, uniqueModifiers);
+        const pressedModifiers = [];
+        let failure;
+        try {
+            for (const modifier of uniqueModifiers){
+                this.keyToggle(modifier, 'down');
+                pressedModifiers.push(modifier);
+                await this.delay(delayMs);
+            }
+            this.keyTap(key);
+            await this.delay(delayMs);
+        } catch (error) {
+            failure = error;
+        }
+        for (const modifier of pressedModifiers.reverse()){
+            const releaseFailure = this.tryReleaseKey(modifier);
+            if (void 0 === failure && void 0 !== releaseFailure) failure = releaseFailure;
+            if (void 0 === failure) try {
+                await this.delay(delayMs);
+            } catch (error) {
+                failure = error;
+            }
+        }
+        if (void 0 !== failure) throw failure;
+    }
     typeString(text) {
         this.getLibnutOrThrow('typeString').typeString(text);
     }
@@ -214,6 +316,20 @@ class ComputerInputDriver {
             this.options.debug(`Failed to release mouse button ${button}: ${error}`);
         }
     }
+    tryReleaseKey(key) {
+        try {
+            const libnut = this.options.getLibnut();
+            external_node_assert_default()(libnut, 'libnut not initialized');
+            libnut.keyToggle(key, 'up');
+            return;
+        } catch (error) {
+            const releaseError = new Error(`Failed to release modifier key "${key}"`, {
+                cause: error
+            });
+            this.options.debug(`${releaseError.message}: ${error}`);
+            return releaseError;
+        }
+    }
     rejectPendingInputDelays() {
         const error = this.createDestroyedError('in-flight input');
         for (const waitRef of this.pendingInputDelayWaits){
@@ -231,6 +347,96 @@ class ComputerInputDriver {
         this.pendingInputDelayWaits = new Set();
     }
 }
+const US_SHIFTED_CHARACTER_KEYS = new Map([
+    [
+        '~',
+        '`'
+    ],
+    [
+        '!',
+        '1'
+    ],
+    [
+        '@',
+        '2'
+    ],
+    [
+        '#',
+        '3'
+    ],
+    [
+        '$',
+        '4'
+    ],
+    [
+        '%',
+        '5'
+    ],
+    [
+        '^',
+        '6'
+    ],
+    [
+        '&',
+        '7'
+    ],
+    [
+        '*',
+        '8'
+    ],
+    [
+        '(',
+        '9'
+    ],
+    [
+        ')',
+        '0'
+    ],
+    [
+        '_',
+        '-'
+    ],
+    [
+        '+',
+        '='
+    ],
+    [
+        '{',
+        '['
+    ],
+    [
+        '}',
+        ']'
+    ],
+    [
+        '|',
+        '\\'
+    ],
+    [
+        ':',
+        ';'
+    ],
+    [
+        '"',
+        "'"
+    ],
+    [
+        '<',
+        ','
+    ],
+    [
+        '>',
+        '.'
+    ],
+    [
+        '?',
+        '/'
+    ]
+]);
+function resolveShiftedKey(character, keyboardLayout) {
+    if (/^[A-Z]$/.test(character)) return character.toLowerCase();
+    return 'en-US' === keyboardLayout ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+}
 const POWERSHELL_TIMEOUT_MS = 15000;
 const POWERSHELL_MAX_BUFFER = 67108864;
 const WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE = `
@@ -595,92 +801,6 @@ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
 const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
 const LIBNUT_FALLBACK_MAX_DETENTS = 200;
 const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
-const LINUX_SHIFTED_CHARACTER_KEYS = new Map([
-    [
-        '~',
-        '`'
-    ],
-    [
-        '!',
-        '1'
-    ],
-    [
-        '@',
-        '2'
-    ],
-    [
-        '#',
-        '3'
-    ],
-    [
-        '$',
-        '4'
-    ],
-    [
-        '%',
-        '5'
-    ],
-    [
-        '^',
-        '6'
-    ],
-    [
-        '&',
-        '7'
-    ],
-    [
-        '*',
-        '8'
-    ],
-    [
-        '(',
-        '9'
-    ],
-    [
-        ')',
-        '0'
-    ],
-    [
-        '_',
-        '-'
-    ],
-    [
-        '+',
-        '='
-    ],
-    [
-        '{',
-        '['
-    ],
-    [
-        '}',
-        ']'
-    ],
-    [
-        '|',
-        '\\'
-    ],
-    [
-        ':',
-        ';'
-    ],
-    [
-        '"',
-        "'"
-    ],
-    [
-        '<',
-        ','
-    ],
-    [
-        '>',
-        '.'
-    ],
-    [
-        '?',
-        '/'
-    ]
-]);
 const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
 const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
 const EDGE_SCROLL_SPEC = {
@@ -717,67 +837,6 @@ const EDGE_SCROLL_SPEC = {
         ]
     }
 };
-const APPLESCRIPT_KEY_CODE_MAP = {
-    return: 36,
-    enter: 36,
-    tab: 48,
-    space: 49,
-    backspace: 51,
-    delete: 51,
-    escape: 53,
-    forwarddelete: 117,
-    left: 123,
-    right: 124,
-    down: 125,
-    up: 126,
-    home: 115,
-    end: 119,
-    pageup: 116,
-    pagedown: 121,
-    f1: 122,
-    f2: 120,
-    f3: 99,
-    f4: 118,
-    f5: 96,
-    f6: 97,
-    f7: 98,
-    f8: 100,
-    f9: 101,
-    f10: 109,
-    f11: 103,
-    f12: 111
-};
-const APPLESCRIPT_MODIFIER_MAP = {
-    command: 'command down',
-    cmd: 'command down',
-    control: 'control down',
-    ctrl: 'control down',
-    shift: 'shift down',
-    alt: 'option down',
-    option: 'option down',
-    meta: 'command down'
-};
-function sendKeyViaAppleScript(key, modifiers = []) {
-    const lowerKey = key.toLowerCase();
-    const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
-    const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
-    const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
-    let script;
-    if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
-    else {
-        const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-        script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
-    }
-    debugDevice('sendKeyViaAppleScript', {
-        key,
-        modifiers,
-        script
-    });
-    (0, external_node_child_process_namespaceObject.execFileSync)("osascript", [
-        '-e',
-        script
-    ]);
-}
 function escapePowershellSingleQuoted(value) {
     return value.replace(/'/g, "''");
 }
@@ -1205,7 +1264,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck(displays) {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.12.7-beta-20260911063016.0");
+        console.log("[HealthCheck] @midscene/computer v1.12.7");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -1382,9 +1441,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             ]);
             else {
                 const modifier = 'darwin' === process.platform ? 'command' : 'control';
-                this.inputDriver.keyTap('v', [
+                await this.inputDriver.keyTapWithModifierDelay('v', [
                     modifier
-                ]);
+                ], this.options?.keyboardModifierDelay ?? 0);
             }
             await this.inputDriver.delay(100);
         } finally{
@@ -1399,12 +1458,16 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
     }
     async typeStringWithDelay(text, keyboardTypeDelay) {
         await (0, device_namespaceObject.sendTextSequentially)(text.replace(/\r\n?/g, '\n'), {
-            sendCharacter: (character)=>{
-                const linuxShiftedKey = 'linux' === process.platform ? LINUX_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+            sendCharacter: async (character)=>{
+                const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
+                const pacedShiftedKey = !this.useAppleScript && (this.options?.keyboardModifierDelay ?? 0) > 0 ? resolveShiftedKey(character, this.options?.keyboardLayout) : void 0;
                 if ('\n' === character) this.inputDriver.sendKey('return');
                 else if ('\t' === character) this.inputDriver.sendKey('tab');
                 else if (' ' === character) this.inputDriver.sendKey('space');
                 else if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript(character);
+                else if (void 0 !== pacedShiftedKey) await this.inputDriver.keyTapWithModifierDelay(pacedShiftedKey, [
+                    'shift'
+                ], this.options?.keyboardModifierDelay ?? 0);
                 else if (void 0 !== linuxShiftedKey) this.inputDriver.keyTap(linuxShiftedKey, [
                     'shift'
                 ]);
@@ -1425,9 +1488,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             return;
         }
         const modifier = 'darwin' === process.platform ? 'command' : 'control';
-        this.inputDriver.keyTap('a', [
+        await this.inputDriver.keyTapWithModifierDelay('a', [
             modifier
-        ]);
+        ], this.options?.keyboardModifierDelay ?? 0);
         await this.inputDriver.delay(50);
         this.inputDriver.keyTap('backspace');
     }
@@ -1441,6 +1504,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
             modifiers,
             driver: this.useAppleScript ? "applescript" : 'libnut'
         });
+        if (!this.useAppleScript && modifiers.length > 0 && (this.options?.keyboardModifierDelay ?? 0) > 0) return void await this.inputDriver.keyTapWithModifierDelay(key, modifiers, this.options?.keyboardModifierDelay ?? 0);
         this.inputDriver.sendKey(key, modifiers);
     }
     resolveUntargetedScrollPoint(screenSize) {
@@ -1594,7 +1658,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
         device_define_property(this, "inputDriver", new ComputerInputDriver({
             getLibnut: ()=>device_libnut,
             useAppleScript: ()=>this.useAppleScript,
-            sendKeyViaAppleScript,
+            sendKeyViaAppleScript: sendKeyViaAppleScript,
             runPhasedScroll,
             debug: (message)=>debugDevice(message)
         }));
@@ -1720,6 +1784,8 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
                 }
             }
         });
+        if (options?.keyboardModifierDelay !== void 0 && (!Number.isFinite(options.keyboardModifierDelay) || options.keyboardModifierDelay < 0)) throw new Error('keyboardModifierDelay must be a finite non-negative number');
+        if (options?.keyboardLayout !== void 0 && 'en-US' !== options.keyboardLayout) throw new Error('keyboardLayout must be "en-US" when specified');
         this.options = options;
         this.displayId = options?.displayId;
         this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -2467,6 +2533,8 @@ function createLocalComputerDevice(opts) {
         keyboardTypeDelay: opts?.keyboardTypeDelay,
         inputStrategy: opts?.inputStrategy,
         keyboardDriver: opts?.keyboardDriver,
+        keyboardModifierDelay: opts?.keyboardModifierDelay,
+        keyboardLayout: opts?.keyboardLayout,
         headless: opts?.headless,
         xvfbResolution: opts?.xvfbResolution,
         keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
@@ -2527,6 +2595,10 @@ const computerInitArgShape = {
     headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
     keyboardTypeDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
     inputStrategy: core_namespaceObject.z["enum"](device_namespaceObject.inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
+    keyboardModifierDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds after each modifier transition and the main key for local libnut keyboard events. Positive values pace shortcuts and the implicit Shift used by uppercase letters during sequential input. Layout-dependent punctuation also requires keyboardLayout. Ignored in RDP mode and by the macOS AppleScript driver.'),
+    keyboardLayout: core_namespaceObject.z["enum"]([
+        'en-US'
+    ]).optional().describe('Keyboard layout used to resolve layout-dependent shifted punctuation to base keys during sequential local libnut input. Currently only "en-US" is supported. Ignored in RDP mode and by the macOS AppleScript driver.'),
     host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
     port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
     username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
@@ -2543,7 +2615,7 @@ const computerInitArgShape = {
 function adaptComputerInitArgs(extracted) {
     if (!extracted || 0 === Object.keys(extracted).length) return;
     if (extracted.host) {
-        const { displayId: _d, headless: _h, ...rdpFields } = extracted;
+        const { displayId: _d, headless: _h, keyboardModifierDelay: _s, keyboardLayout: _l, ...rdpFields } = extracted;
         const host = normalizeRdpHost(extracted.host);
         return {
             mode: 'rdp',
@@ -2557,6 +2629,8 @@ function adaptComputerInitArgs(extracted) {
         headless: extracted.headless,
         keyboardTypeDelay: extracted.keyboardTypeDelay,
         inputStrategy: extracted.inputStrategy,
+        keyboardModifierDelay: extracted.keyboardModifierDelay,
+        keyboardLayout: extracted.keyboardLayout,
         ...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(extracted) ?? {}
     };
 }
@@ -2608,6 +2682,8 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
         const headless = opts?.mode === 'local' ? opts.headless : void 0;
         const keyboardTypeDelay = opts?.keyboardTypeDelay;
         const inputStrategy = opts?.inputStrategy;
+        const keyboardModifierDelay = opts?.mode === 'local' ? opts.keyboardModifierDelay : void 0;
+        const keyboardLayout = opts?.mode === 'local' ? opts.keyboardLayout : void 0;
         agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
         const agentOpts = {
             ...displayId ? {
@@ -2622,6 +2698,12 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
             ...void 0 !== inputStrategy ? {
                 inputStrategy
             } : {},
+            ...void 0 !== keyboardModifierDelay ? {
+                keyboardModifierDelay
+            } : {},
+            ...void 0 !== keyboardLayout ? {
+                keyboardLayout
+            } : {},
             ...this.options.keepXvfbAliveUntilProcessExit ? {
                 keepXvfbAliveUntilProcessExit: true
             } : {},
@@ -2703,7 +2785,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
 }
 const env_namespaceObject = require("@midscene/shared/env");
 function version() {
-    const currentVersion = "1.12.7-beta-20260911063016.0";
+    const currentVersion = "1.12.7";
     console.log(`@midscene/computer v${currentVersion}`);
     return currentVersion;
 }

package/dist/types/index.d.ts

@@ -177,6 +177,25 @@ export declare interface ComputerDeviceOpt extends ComputerDeviceInputOpt {
      * - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps)
      */
     keyboardDriver?: 'applescript' | 'libnut';
+    /**
+     * Delay in milliseconds after each explicit modifier transition and the
+     * main key for local libnut keyboard events. A positive value changes
+     * modified shortcuts and shifted en-US text characters into separately
+     * observable modifier-down, main-key, and modifier-up phases. This can help
+     * foreground clients whose full-screen keyboard capture misses rapidly
+     * synthesized modifier state changes.
+     *
+     * Ignored by the macOS AppleScript driver and RDP mode.
+     * @default 0
+     */
+    keyboardModifierDelay?: number;
+    /**
+     * Keyboard layout used to resolve layout-dependent shifted characters
+     * during sequential local libnut input. Uppercase Latin letters do not
+     * require this option. Other characters keep using the backend's existing
+     * input behavior unless a supported layout is declared.
+     */
+    keyboardLayout?: 'en-US';
     /**
      * Headless mode via Xvfb (Linux only).
      * - true: start Xvfb virtual display
@@ -210,7 +229,7 @@ export declare type ComputerInterface = ComputerDevice | RDPDevice;
 /** Init args for the local desktop agent (macOS/Windows/Linux). */
 declare type ComputerLocalInitArgs = {
     mode: 'local';
-} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay'> & AgentBehaviorInitArgs;
+} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay' | 'keyboardModifierDelay' | 'keyboardLayout'> & AgentBehaviorInitArgs;
 
 /**
  * Computer-specific tools manager

package/native/rdp/CMakeLists.txt

@@ -92,6 +92,28 @@ if(BUILD_TESTING)
     NAME rdp-session-policy-test
     COMMAND rdp-session-policy-test
   )
+
+  add_executable(
+    rdp-screenshot-test
+    tests/screenshot_test.cpp
+    src/png.cpp
+    src/session.cpp
+  )
+  target_include_directories(
+    rdp-screenshot-test PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${FREERDP_INCLUDE_DIRS}
+  )
+  target_link_libraries(
+    rdp-screenshot-test PRIVATE
+    ${FREERDP_LIBRARIES}
+    Threads::Threads
+  )
+  if(APPLE)
+    target_compile_definitions(rdp-screenshot-test PRIVATE _DARWIN_C_SOURCE)
+  endif()
+  add_test(NAME rdp-screenshot-test COMMAND rdp-screenshot-test)
+  set_tests_properties(rdp-screenshot-test PROPERTIES TIMEOUT 30)
 endif()
 
 if(APPLE)

package/native/rdp/include/rdp_helper_framebuffer.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+
+#include "rdp_helper_session.hpp"
+
+namespace midscene::rdp {
+
+// FreeRDP embeds its context first; shared by the callback and native tests.
+struct MidsceneRdpContext {
+  rdpContext context;
+  FreeRdpSessionTransport* owner = nullptr;
+};
+
+// Compute the next initial-screenshot wakeup without consulting wall time.
+inline std::chrono::steady_clock::time_point ScreenshotWakeAt(
+    std::chrono::steady_clock::time_point started,
+    std::chrono::steady_clock::time_point last_update) {
+  return std::min(std::max(started, last_update) + std::chrono::milliseconds(300),
+                  started + std::chrono::seconds(3));
+}
+
+}  // namespace midscene::rdp

package/native/rdp/include/rdp_helper_session.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <atomic>
+#include <chrono>
 #include <condition_variable>
 #include <cstdint>
 #include <mutex>
@@ -76,7 +77,7 @@ class FreeRdpSessionTransport final : public SessionTransport {
   void HookEndPaint(rdpUpdate* update);
   // Records that at least one paint has reached the local framebuffer and
   // wakes Connect().
-  void MarkFramePainted(std::optional<RawFrame> first_frame = std::nullopt);
+  void MarkFramePainted();
   // Records a framebuffer update even when it is too blank or uniform to count
   // as the first informative frame.
   void MarkFramebufferUpdated();
@@ -90,6 +91,7 @@ class FreeRdpSessionTransport final : public SessionTransport {
 
  private:
   friend struct MidsceneRdpContext;
+  friend struct RdpScreenshotTestPeer;
 
   freerdp* instance_ = nullptr;
   std::thread event_thread_;
@@ -110,8 +112,10 @@ class FreeRdpSessionTransport final : public SessionTransport {
   std::atomic<bool> session_active_{false};
   std::condition_variable frame_cv_;
   std::mutex frame_mutex_;
-  std::optional<RawFrame> first_frame_;
-  bool first_frame_consumed_ = false;
+  // Protected by frame_mutex_; use a monotonic clock for screenshot settling.
+  std::chrono::steady_clock::time_point last_frame_update_{};
+  // Protected by mutex_; reset for each new connection.
+  bool first_screenshot_pending_ = true;
   pEndPaint original_end_paint_ = nullptr;
   uint16_t mouse_x_ = 0;
   uint16_t mouse_y_ = 0;

package/native/rdp/src/png.cpp

@@ -81,12 +81,15 @@ std::vector<uint8_t> BuildImageData(const RawFrame& frame) {
       const uint8_t blue = frame.bgra[pixel_offset];
       const uint8_t green = frame.bgra[pixel_offset + 1];
       const uint8_t red = frame.bgra[pixel_offset + 2];
-      const uint8_t alpha = frame.bgra[pixel_offset + 3];
 
       image_data.push_back(red);
       image_data.push_back(green);
       image_data.push_back(blue);
-      image_data.push_back(alpha);
+      // FreeRDP's PIXEL_FORMAT_BGRA32 is an opaque BGRX desktop surface. The
+      // fourth byte is padding and may be zero, so treating it as alpha makes
+      // otherwise valid screenshots transparent and turns them black during
+      // PNG-to-JPEG conversion.
+      image_data.push_back(0xFF);
     }
   }