@loadmill/droid-cua 2.39.0 → 2.41.0

package/build/src/desktop/computer-use.js

@@ -1,6 +1,9 @@
 import { buildActionEventMeta } from "../device/action-event-meta.js";
 import { buildTypeActionPayload, buildTypeActionText, } from "../device/sensitive-action.js";
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+import { getWindowsVisibleBounds } from "./windows-visible-bounds.js";
+import sharp from "sharp";
+import { capturePrimaryDisplay } from "./primary-display.js";
 const SCREENSHOT_WIDTH = 1440;
 const ACCESSIBILITY_HIT_TOLERANCE = 12;
 const MACOS_CAPTURE_INSETS = { left: 50, right: 50, top: 38, bottom: 62 };
@@ -26,6 +29,57 @@ function assertResult(result, label) {
         throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
     }
 }
+function readScreenshot(result) {
+    const image = result.content?.find((item) => item.type === "image");
+    const sizeText = result.content?.find((item) => item.type === "text")?.text || "";
+    const size = /^(\d+)x(\d+)$/.exec(sizeText.trim());
+    if (!image?.data || !size)
+        throw new Error("Desktop screenshot returned no image or dimensions.");
+    return { data: image.data, width: Number(size[1]), height: Number(size[2]) };
+}
+async function isBlackScreenshot(data) {
+    const { channels } = await sharp(Buffer.from(data, "base64")).stats();
+    return channels.slice(0, 3).every((channel) => channel.max === 0);
+}
+async function captureWindowFromDesktop(native, bounds) {
+    const display = native.getDisplaySize();
+    // Read full-resolution pixels; resizing the desktop first loses window detail.
+    const screen = native.takeScreenshot(undefined, undefined, 0, undefined, undefined);
+    if (!screen.base64)
+        throw new Error("Desktop screen capture returned no image.");
+    const left = Math.round(bounds.x * screen.width / display.width);
+    const top = Math.round(bounds.y * screen.height / display.height);
+    const width = Math.round(bounds.width * screen.width / display.width);
+    const height = Math.round(bounds.height * screen.height / display.height);
+    const visibleLeft = Math.max(0, left);
+    const visibleTop = Math.max(0, top);
+    const visibleRight = Math.min(screen.width, left + width);
+    const visibleBottom = Math.min(screen.height, top + height);
+    if (visibleRight <= visibleLeft || visibleBottom <= visibleTop) {
+        throw new Error("The window must overlap the primary display for Windows screen capture.");
+    }
+    let image = sharp(Buffer.from(screen.base64, "base64")).extract({
+        left: visibleLeft,
+        top: visibleTop,
+        width: visibleRight - visibleLeft,
+        height: visibleBottom - visibleTop,
+    });
+    if (visibleLeft !== left || visibleTop !== top || visibleRight !== left + width || visibleBottom !== top + height) {
+        image = image.extend({
+            left: visibleLeft - left,
+            top: visibleTop - top,
+            right: left + width - visibleRight,
+            bottom: top + height - visibleBottom,
+            background: "black",
+        });
+    }
+    // Materialize the crop and padding before resize (Sharp can reorder operations).
+    const cropped = await image.png().toBuffer();
+    const { data, info } = await sharp(cropped)
+        .resize({ width: SCREENSHOT_WIDTH, withoutEnlargement: true })
+        .png().toBuffer({ resolveWithObject: true });
+    return { data: data.toString("base64"), width: info.width, height: info.height };
+}
 function sameBounds(left, right) {
     return left && right && left.x === right.x && left.y === right.y && left.width === right.width && left.height === right.height;
 }
@@ -51,7 +105,7 @@ function formatPermissionError(error, platform) {
     if (platform === "darwin" && /permission|not authorized|screencapture|screen recording|cgwindow|axerror/i.test(message)) {
         return `${message} Grant Screen & System Audio Recording and Accessibility access to Droid (or your terminal for CLI runs) in System Settings > Privacy & Security, then restart it.`;
     }
-    if (platform === "win32" && /access denied|elevat|integrity level|could not focus/i.test(message)) {
+    if (platform === "win32" && /access denied|elevat|integrity level/i.test(message)) {
         return `${message} Run Droid in an interactive desktop session and at the same integrity level as the target application; elevated Windows applications cannot be controlled by a non-elevated Droid process.`;
     }
     return message;
@@ -63,7 +117,7 @@ async function createLibraryClient() {
     ]);
     return connectInProcess(createComputerUseServer({ profile: "ax", vision: true }));
 }
-async function loadAccessibilityDriver() {
+async function loadNativeDriver() {
     const { loadNative } = await import("@zavora-ai/computer-use-mcp/host-native");
     return loadNative();
 }
@@ -183,102 +237,214 @@ function normalizeShortcut(keys, platform) {
     return keys.map((key) => aliases[String(key).toUpperCase()] || String(key).toLowerCase()).join("+");
 }
 /**
- * @param {{ app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", client?: any, accessibility?: any }} options
+ * @param {{ scope?: "window" | "desktop", app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", focusOnConnect?: boolean, client?: any, accessibility?: any, visibleBoundsReader?: typeof getWindowsVisibleBounds, displayCapture?: typeof capturePrimaryDisplay }} options
  */
-export async function createDesktopController({ app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", client: injectedClient = null, accessibility: injectedAccessibility = null, } = {}) {
+export async function createDesktopController({ scope = "window", app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", focusOnConnect = false, client: injectedClient = null, accessibility: injectedAccessibility = null, visibleBoundsReader = getWindowsVisibleBounds, displayCapture = capturePrimaryDisplay, } = {}) {
     if (!SUPPORTED_PLATFORMS.has(platform)) {
         throw new Error("Desktop control is currently supported only on macOS and Windows.");
     }
-    if (!requestedApp)
+    if (scope !== "window" && scope !== "desktop")
+        throw new Error('Desktop scope must be "window" or "desktop".');
+    if (scope === "desktop" && (requestedApp || requestedWindowId != null))
+        throw new Error("Full desktop cannot select an application or window.");
+    if (scope === "desktop" && inputMode !== "physical")
+        throw new Error("Full desktop requires physical input.");
+    if (scope === "window" && !requestedApp)
         throw new Error("--desktop-app is required with --target desktop.");
     if (!INPUT_MODES.has(inputMode))
         throw new Error('Desktop input mode must be "accessibility" or "physical".');
     const client = injectedClient || await createLibraryClient();
     try {
-        const apps = (await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid));
-        const application = chooseApplication(apps, requestedApp);
-        const windows = readWindows(await client.listWindows(application.bundleId));
-        let targetWindow = chooseWindow(windows, requestedWindowId == null ? null : Number(requestedWindowId));
+        const application = scope === "window" ? chooseApplication((await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid)), requestedApp) : null;
+        let targetWindow = application ? chooseWindow(readWindows(await client.listWindows(application.bundleId)), requestedWindowId == null ? null : Number(requestedWindowId)) : null;
         let capture = null;
         let deviceInfo = null;
-        let accessibility = injectedAccessibility;
+        let native = injectedAccessibility;
+        let lastVisibleBounds = null;
+        let fallbackLogged = false;
         async function getWindow() {
             return parseJsonResult(await client.getWindow(targetWindow.windowId), "Window lookup");
         }
-        async function screenshot() {
-            let window;
-            let result;
-            try {
-                const currentWindows = readWindows(await client.listWindows(application.bundleId));
-                const focusedWindow = currentWindows.find((candidate) => candidate.isFocused && candidate.isOnScreen);
-                if (focusedWindow)
-                    targetWindow = focusedWindow;
+        async function refreshWindow() {
+            const windows = readWindows(await client.listWindows(application.bundleId))
+                .filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+            const selected = windows.find((window) => window.isFocused)
+                || windows.find((window) => window.windowId === targetWindow.windowId)
+                || chooseWindow(windows, null);
+            if (selected.windowId !== targetWindow.windowId)
+                capture = null;
+            targetWindow = selected;
+            if (deviceInfo) {
+                deviceInfo.desktopWindowId = selected.windowId;
+                deviceInfo.device_name = `${application.displayName} — ${selected.title || "Untitled"}`;
+            }
+        }
+        async function activateWindow() {
+            const activation = parseJsonResult(await client.activateWindow(targetWindow.windowId, 2000), "Window activation");
+            if (!activation.activated || !(await getWindow()).isFocused) {
+                throw new Error(`Droid could not focus the selected desktop window, so no input was sent.${activation.reason ? ` Reason: ${activation.reason}.` : ""}`);
+            }
+        }
+        async function focus() {
+            if (scope === "desktop")
+                return;
+            await refreshWindow();
+            await activateWindow();
+        }
+        async function captureFromDesktop(window) {
+            if (!window.isFocused) {
+                if (inputMode !== "physical") {
+                    throw new Error("Windows desktop capture requires the target window to be visible and focused. Bring it to the foreground and retry.");
+                }
+                await activateWindow();
                 window = await getWindow();
-                if (!window.isOnScreen)
-                    throw new Error("The selected desktop window is no longer visible.");
-                result = await client.screenshot({
-                    width: SCREENSHOT_WIDTH,
-                    quality: 0,
-                    target_window_id: targetWindow.windowId,
-                });
-                assertResult(result, "Desktop screenshot");
+            }
+            native ||= await loadNativeDriver();
+            if (lastVisibleBounds?.windowId !== targetWindow.windowId || !sameBounds(lastVisibleBounds.outer, window.bounds)) {
+                lastVisibleBounds = {
+                    windowId: targetWindow.windowId,
+                    outer: { ...window.bounds },
+                    visible: await visibleBoundsReader(targetWindow.windowId, window.bounds),
+                };
+            }
+            const actionBounds = lastVisibleBounds.visible;
+            const image = await captureWindowFromDesktop(native, actionBounds);
+            if (!fallbackLogged) {
+                console.warn("Windows window capture returned all-black pixels; using a visible-desktop crop. Keep the target unobstructed on the primary display.");
+                fallbackLogged = true;
+            }
+            return { ...image, bounds: { ...window.bounds }, actionBounds: { ...actionBounds } };
+        }
+        async function screenshot({ allowDesktopFallback = true } = {}) {
+            try {
+                let image;
+                if (scope === "desktop") {
+                    native ||= await loadNativeDriver();
+                    const screen = await displayCapture(native, platform);
+                    image = { ...screen, actionBounds: { ...screen.bounds } };
+                }
+                else {
+                    await refreshWindow();
+                    const window = await getWindow();
+                    if (!window.isOnScreen)
+                        throw new Error("The selected desktop window is no longer visible.");
+                    const result = await client.screenshot({
+                        width: SCREENSHOT_WIDTH,
+                        quality: 0,
+                        target_window_id: targetWindow.windowId,
+                    });
+                    assertResult(result, "Desktop screenshot");
+                    image = { ...readScreenshot(result), bounds: { ...window.bounds }, actionBounds: { ...window.bounds } };
+                    if (allowDesktopFallback && platform === "win32" && await isBlackScreenshot(image.data)) {
+                        image = await captureFromDesktop(window);
+                    }
+                }
+                capture = {
+                    bounds: image.bounds,
+                    actionBounds: image.actionBounds,
+                    width: image.width,
+                    height: image.height,
+                    windowId: targetWindow?.windowId ?? null,
+                    displayId: image.displayId,
+                    contentBounds: scope === "desktop"
+                        ? { x: 0, y: 0, width: image.width, height: image.height }
+                        : getScreenshotContentBounds(image.width, image.height, image.bounds, platform),
+                };
+                if (deviceInfo) {
+                    deviceInfo.device_width = image.actionBounds.width;
+                    deviceInfo.device_height = image.actionBounds.height;
+                    deviceInfo.scaled_width = image.width;
+                    deviceInfo.scaled_height = image.height;
+                    deviceInfo.scale = image.width / image.actionBounds.width;
+                    deviceInfo.desktopWindowId = targetWindow?.windowId ?? null;
+                }
+                return image.data;
             }
             catch (error) {
                 throw new Error(formatPermissionError(error, platform));
             }
-            const image = result.content?.find((item) => item.type === "image");
-            const sizeText = result.content?.find((item) => item.type === "text")?.text || "";
-            const size = /^(\d+)x(\d+)$/.exec(sizeText.trim());
-            if (!image?.data || !size)
-                throw new Error("Desktop screenshot returned no image or dimensions.");
-            capture = {
-                bounds: { ...window.bounds },
-                width: Number(size[1]),
-                height: Number(size[2]),
-            };
-            capture.contentBounds = getScreenshotContentBounds(capture.width, capture.height, window.bounds, platform);
-            if (deviceInfo) {
-                deviceInfo.device_width = window.bounds.width;
-                deviceInfo.device_height = window.bounds.height;
-                deviceInfo.scaled_width = capture.width;
-                deviceInfo.scaled_height = capture.height;
-                deviceInfo.scale = capture.width / window.bounds.width;
-                deviceInfo.desktopWindowId = targetWindow.windowId;
-            }
-            return image.data;
         }
         function scaledPoint(x, y) {
             if (!capture)
                 throw new Error("Capture the desktop window before executing an action.");
+            if (scope === "desktop" && (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x >= capture.width || y >= capture.height)) {
+                throw new Error("The action is outside the primary display.");
+            }
             const content = capture.contentBounds;
-            return [
-                Math.round(capture.bounds.x + ((x - content.x) * capture.bounds.width) / content.width),
-                Math.round(capture.bounds.y + ((y - content.y) * capture.bounds.height) / content.height),
+            const bounds = capture.actionBounds;
+            const point = [
+                Math.round(bounds.x + ((x - content.x) * bounds.width) / content.width),
+                Math.round(bounds.y + ((y - content.y) * bounds.height) / content.height),
             ];
+            // Retina rounding at the last image pixel must stay inside the display.
+            return scope === "desktop" ? [Math.min(point[0], bounds.width - 1), Math.min(point[1], bounds.height - 1)] : point;
         }
         async function prepareAction(requireFocus) {
             if (!capture)
-                throw new Error("Capture the desktop window before executing an action.");
-            const before = await getWindow();
-            if (!sameBounds(before.bounds, capture.bounds)) {
+                throw new Error("A fresh desktop screenshot is required before acting.");
+            if (scope === "desktop") {
+                const display = native.getDisplaySize(capture.displayId);
+                if (display.width !== capture.bounds.width || display.height !== capture.bounds.height) {
+                    throw new Error("The display resized after the screenshot. A fresh screenshot is required before acting.");
+                }
+                return;
+            }
+            let window;
+            try {
+                window = parseJsonResult(await client.getWindow(capture.windowId), "Window lookup");
+            }
+            catch {
+                throw new Error("The captured desktop window is no longer available. A fresh screenshot is required before acting.");
+            }
+            if (!window.isOnScreen || window.bundleId !== application.bundleId || capture.windowId !== targetWindow.windowId) {
+                throw new Error("The captured desktop window is no longer available. A fresh screenshot is required before acting.");
+            }
+            if (!sameBounds(window.bounds, capture.bounds)) {
                 throw new Error("The desktop window moved or resized after the screenshot. A fresh screenshot is required before acting.");
             }
-            if (!requireFocus)
-                return;
-            const activation = parseJsonResult(await client.activateWindow(targetWindow.windowId, 2000), "Window activation");
-            const focused = await getWindow();
-            if (!activation.activated || !focused.isFocused) {
-                throw new Error("Droid could not focus the selected desktop window, so no input was sent.");
+            if (requireFocus) {
+                await activateWindow();
+                if (!sameBounds((await getWindow()).bounds, capture.bounds)) {
+                    throw new Error("The desktop window moved or resized during focus. A fresh screenshot is required before acting.");
+                }
             }
         }
-        const targetOptions = () => ({ targetWindowId: targetWindow.windowId, focusStrategy: "strict" });
+        const targetOptions = () => scope === "desktop"
+            ? { focusStrategy: "none" }
+            : { targetWindowId: targetWindow.windowId, focusStrategy: "strict" };
+        // Native pointer calls avoid the library's first-monitor coordinate validation.
+        // Keyboard calls retain its text/clipboard handling, with app focus disabled.
+        const physical = scope === "window" ? client : {
+            click: (x, y) => native.mouseClick(x, y, "left", 1),
+            doubleClick: (x, y) => native.mouseClick(x, y, "left", 2),
+            moveMouse: (x, y) => native.mouseMove(x, y),
+            drag: async (to, from) => {
+                native.mouseMove(from[0], from[1]);
+                await new Promise((resolve) => setTimeout(resolve, 50));
+                native.mouseButton("press", from[0], from[1]);
+                try {
+                    await new Promise((resolve) => setTimeout(resolve, 50));
+                    native.mouseDrag(to[0], to[1]);
+                    await new Promise((resolve) => setTimeout(resolve, 50));
+                }
+                finally {
+                    native.mouseButton("release", to[0], to[1]);
+                }
+            },
+            scroll: (x, y, direction, amount) => {
+                native.mouseMove(x, y);
+                native.mouseScroll(direction === "down" ? amount : direction === "up" ? -amount : 0, direction === "right" ? amount : direction === "left" ? -amount : 0);
+            },
+            type: client.type,
+            key: client.key,
+        };
         async function performCursorFreeAction(actionType, x, y, direction = null) {
             const tree = parseJsonResult(await client.getUiTree(targetWindow.windowId, 20), "Accessibility tree");
             const selected = chooseAccessibilityElement(tree, x, y, actionType, direction);
             if (!selected)
                 throw new Error(cursorFreeError(actionType));
-            accessibility ||= await loadAccessibilityDriver();
-            const result = accessibility.performAction(targetWindow.windowId, selected.element.role, selected.element.label, selected.action);
+            native ||= await loadNativeDriver();
+            const result = native.performAction(targetWindow.windowId, selected.element.role, selected.element.label, selected.action);
             if (!result?.performed)
                 throw new Error(`${cursorFreeError(actionType)}${result?.reason ? ` Reason: ${result.reason}.` : ""}`);
         }
@@ -304,8 +470,8 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                         }
                         else {
                             const result = action.type === "click"
-                                ? await client.click(x, y, application.bundleId, targetOptions())
-                                : await client.doubleClick(x, y, application.bundleId, targetOptions());
+                                ? await physical.click(x, y, application?.bundleId, targetOptions())
+                                : await physical.doubleClick(x, y, application?.bundleId, targetOptions());
                             assertResult(result, "Desktop click");
                         }
                         break;
@@ -315,7 +481,7 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                         addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
                         if (inputMode === "accessibility")
                             throw new Error(cursorFreeError(action.type));
-                        assertResult(await client.moveMouse(x, y, application.bundleId, targetOptions()), "Desktop mouse move");
+                        assertResult(await physical.moveMouse(x, y, application?.bundleId, targetOptions()), "Desktop mouse move");
                         break;
                     }
                     case "drag": {
@@ -327,7 +493,7 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                         addOutput({ type: "action", text: `Dragging from (${from[0]}, ${from[1]}) to (${to[0]}, ${to[1]})`, ...meta({ pathStart: { x: from[0], y: from[1] }, pathEnd: { x: to[0], y: to[1] }, unit: "px" }) });
                         if (inputMode === "accessibility")
                             throw new Error(cursorFreeError(action.type));
-                        assertResult(await client.drag(to, from, application.bundleId, targetOptions()), "Desktop drag");
+                        assertResult(await physical.drag(to, from, application?.bundleId, targetOptions()), "Desktop drag");
                         break;
                     }
                     case "scroll": {
@@ -342,20 +508,20 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                             await performCursorFreeAction(action.type, x, y, direction);
                         }
                         else {
-                            assertResult(await client.scroll(x, y, direction, amount, application.bundleId, targetOptions()), "Desktop scroll");
+                            assertResult(await physical.scroll(x, y, direction, amount, application?.bundleId, targetOptions()), "Desktop scroll");
                         }
                         break;
                     }
                     case "type": {
                         const text = typeof action.text === "string" ? action.text : "";
                         addOutput({ type: "action", text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
-                        assertResult(await client.type(text, application.bundleId, targetOptions()), "Desktop typing");
+                        assertResult(await physical.type(text, application?.bundleId, targetOptions()), "Desktop typing");
                         break;
                     }
                     case "keypress": {
                         const shortcut = normalizeShortcut(action.keys, platform);
                         addOutput({ type: "action", text: `Pressing key: ${shortcut}`, ...meta({ keys: action.keys, shortcut }) });
-                        assertResult(await client.key(shortcut, application.bundleId, targetOptions()), "Desktop keypress");
+                        assertResult(await physical.key(shortcut, application?.bundleId, targetOptions()), "Desktop keypress");
                         break;
                     }
                     case "wait": {
@@ -375,26 +541,31 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                 return { success: false, error: message };
             }
         }
-        await screenshot();
+        if (focusOnConnect)
+            await focus();
+        // Without focus, connection only probes the window; the desktop may be covered.
+        await screenshot({ allowDesktopFallback: focusOnConnect });
         deviceInfo = {
             platform: "desktop",
             source: "local",
-            device_name: `${application.displayName} — ${targetWindow.title || "Untitled"}`,
-            device_width: targetWindow.bounds.width,
-            device_height: targetWindow.bounds.height,
+            device_name: application ? `${application.displayName} — ${targetWindow.title || "Untitled"}` : "Full desktop",
+            device_width: capture.actionBounds.width,
+            device_height: capture.actionBounds.height,
             scaled_width: capture.width,
             scaled_height: capture.height,
-            scale: capture.width / targetWindow.bounds.width,
-            desktopAppId: application.bundleId,
-            desktopWindowId: targetWindow.windowId,
+            scale: capture.width / capture.actionBounds.width,
+            desktopScope: scope,
+            desktopAppId: application?.bundleId ?? null,
+            desktopWindowId: targetWindow?.windowId ?? null,
             desktopInputMode: inputMode,
         };
         return {
             application,
-            window: targetWindow,
-            deviceId: `desktop:${targetWindow.windowId}`,
+            get window() { return targetWindow; },
+            deviceId: scope === "desktop" ? "desktop:primary" : `desktop:${targetWindow.windowId}`,
             deviceInfo,
-            getScreenshotAsBase64: screenshot,
+            focus,
+            getScreenshotAsBase64: () => screenshot(),
             handleModelAction: handleAction,
             close: () => client.close(),
         };

package/build/src/desktop/primary-display.js

@@ -0,0 +1,74 @@
+import { execFile } from "node:child_process";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import sharp from "sharp";
+const runFile = promisify(execFile);
+const WINDOWS_CAPTURE = `
+$ErrorActionPreference = 'Stop'
+Add-Type -AssemblyName System.Drawing
+Add-Type -TypeDefinition @'
+using System;
+using System.Runtime.InteropServices;
+public static class PrimaryDisplay {
+  [StructLayout(LayoutKind.Sequential)] public struct Point { public int X; public int Y; }
+  [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
+  [DllImport("user32.dll")] public static extern int GetSystemMetrics(int index);
+  [DllImport("user32.dll")] public static extern IntPtr MonitorFromPoint(Point point, uint flags);
+}
+'@
+[void][PrimaryDisplay]::SetProcessDPIAware()
+$width = [PrimaryDisplay]::GetSystemMetrics(0)
+$height = [PrimaryDisplay]::GetSystemMetrics(1)
+$bitmap = [System.Drawing.Bitmap]::new($width, $height)
+$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+try {
+  $graphics.CopyFromScreen(0, 0, 0, 0, $bitmap.Size)
+  $bitmap.Save('CAPTURE_PATH', [System.Drawing.Imaging.ImageFormat]::Png)
+  # The native driver's display lookup accepts a uint32 monitor handle.
+  [PrimaryDisplay]::MonitorFromPoint([PrimaryDisplay+Point]::new(), 1).ToInt64() -band 0xffffffff
+} finally {
+  $graphics.Dispose()
+  $bitmap.Dispose()
+}
+`;
+/** Capture only the primary display, independently of the library's cached window target. */
+export async function capturePrimaryDisplay(native, platform = process.platform, run = runFile) {
+    const directory = await mkdtemp(join(tmpdir(), "droid-desktop-"));
+    const path = join(directory, "screen.png");
+    try {
+        let displayId;
+        if (platform === "darwin") {
+            await run("/usr/sbin/screencapture", ["-x", "-m", "-t", "png", path], { timeout: 10000 });
+        }
+        else if (platform === "win32") {
+            const script = WINDOWS_CAPTURE.replace("CAPTURE_PATH", path.replaceAll("'", "''"));
+            const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], {
+                windowsHide: true,
+                timeout: 10000,
+            });
+            displayId = Number(stdout.trim());
+            if (!Number.isSafeInteger(displayId) || displayId <= 0)
+                throw new Error("Primary display lookup failed.");
+        }
+        else {
+            throw new Error("Desktop control is currently supported only on macOS and Windows.");
+        }
+        // Use the native driver's coordinate units, which can differ from screenshot pixels.
+        const display = native.getDisplaySize(displayId);
+        const { data, info } = await sharp(await readFile(path))
+            .resize({ width: 1440, withoutEnlargement: true })
+            .png().toBuffer({ resolveWithObject: true });
+        return {
+            data: data.toString("base64"),
+            width: info.width,
+            height: info.height,
+            bounds: { x: 0, y: 0, width: display.width, height: display.height },
+            displayId: displayId ?? display.displayId,
+        };
+    }
+    finally {
+        await rm(directory, { recursive: true, force: true });
+    }
+}

package/build/src/desktop/windows-visible-bounds.js

@@ -0,0 +1,55 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+const runFile = promisify(execFile);
+const RECT_SCRIPT = `
+Add-Type -TypeDefinition @'
+using System;
+using System.Runtime.InteropServices;
+public static class WindowFrame {
+  public const int DWMWA_EXTENDED_FRAME_BOUNDS = 9;
+  [StructLayout(LayoutKind.Sequential)] public struct Rect { public int Left; public int Top; public int Right; public int Bottom; }
+  [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr handle, out Rect rect);
+  [DllImport("dwmapi.dll")] public static extern int DwmGetWindowAttribute(IntPtr handle, int attribute, out Rect rect, int size);
+}
+'@
+$handle = [IntPtr]::new([long]WINDOW_ID)
+$outer = [WindowFrame+Rect]::new()
+$visible = [WindowFrame+Rect]::new()
+if (-not [WindowFrame]::GetWindowRect($handle, [ref]$outer)) { exit 1 }
+$result = [WindowFrame]::DwmGetWindowAttribute($handle, [WindowFrame]::DWMWA_EXTENDED_FRAME_BOUNDS, [ref]$visible, 16)
+if ($result -ne 0) { exit 1 }
+@($outer.Left, $outer.Top, $outer.Right, $outer.Bottom, $visible.Left, $visible.Top, $visible.Right, $visible.Bottom) -join ','
+`;
+export async function getWindowsVisibleBounds(windowId, bounds) {
+    if (process.platform !== "win32")
+        return bounds;
+    if (!Number.isSafeInteger(windowId))
+        return bounds;
+    try {
+        // GetWindowRect includes invisible resize borders; DWM reports the pixels actually drawn.
+        const script = RECT_SCRIPT.replace("WINDOW_ID", String(windowId));
+        const { stdout } = await runFile("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], {
+            windowsHide: true,
+            timeout: 5000,
+        });
+        const [outerLeft, outerTop, outerRight, outerBottom, left, top, right, bottom] = stdout.trim().split(",").map(Number);
+        const scaleX = bounds.width / (outerRight - outerLeft);
+        const scaleY = bounds.height / (outerBottom - outerTop);
+        const insetLeft = Math.round((left - outerLeft) * scaleX);
+        const insetTop = Math.round((top - outerTop) * scaleY);
+        const insetRight = Math.round((outerRight - right) * scaleX);
+        const insetBottom = Math.round((outerBottom - bottom) * scaleY);
+        if (![insetLeft, insetTop, insetRight, insetBottom].every((value) => Number.isFinite(value) && value >= 0)) {
+            throw new Error("Invalid window frame insets.");
+        }
+        const width = bounds.width - insetLeft - insetRight;
+        const height = bounds.height - insetTop - insetBottom;
+        if (width <= 0 || height <= 0)
+            throw new Error("Invalid visible window dimensions.");
+        return { x: bounds.x + insetLeft, y: bounds.y + insetTop, width, height };
+    }
+    catch {
+        console.warn("Windows visible-frame lookup failed; using outer window bounds, which may include invisible resize borders.");
+        return bounds;
+    }
+}