@loadmill/droid-cua 2.34.0 → 2.35.0

package/README.md

@@ -79,6 +79,7 @@ Desktop app support:
 
 **All platforms:**
 - Loadmill sign-in for the Loadmill models, or an OpenAI API key for the OpenAI models
+- Node.js 20 or newer for CLI use
 
 **Android:**
 - Android Debug Bridge (ADB)
@@ -180,6 +181,15 @@ droid-cua --avd adb:emulator-5554 --instructions tests
 # Headless iOS simulator run
 droid-cua --platform ios --avd "iPhone 16" --instructions tests/login.dcua
 
+# List running desktop applications and their window IDs (macOS or Windows)
+droid-cua --target desktop --list-desktop-apps
+
+# Control one already-running desktop application window
+droid-cua --target desktop --desktop-app com.apple.TextEdit --desktop-window 123 --instructions tests/editor.dcua
+
+# Explicitly use cursor-free accessibility input instead of the default physical input
+droid-cua --target desktop --desktop-app com.apple.TextEdit --desktop-input accessibility --instructions tests/editor.dcua
+
 # Headless run with prompt-parity config
 droid-cua --avd adb:emulator-5554 --instructions tests/login.dcua --config ci/droid-cua.json
 
@@ -246,6 +256,11 @@ Supported CLI options include:
 - `--app`
 - `--list-devices`
 - `--browser`
+- `--target` (`mobile`, `web`, or `desktop`)
+- `--desktop-app` (desktop application ID or exact name)
+- `--desktop-window` (required when the application has multiple visible windows)
+- `--desktop-input` (`physical` by default; use `accessibility` for cursor-free input)
+- `--list-desktop-apps`
 - `--instructions`
 - `--config`
 - `--cua-model`
@@ -268,6 +283,8 @@ Supported CLI options include:
 `gpt-5.6-terra`, `gpt-5.6-luna`, and `gpt-5.4`. OpenAI models use direct OpenAI
 mode and require `OPENAI_API_KEY`.
 
+Desktop control supports macOS and Windows. In the desktop app, enable **Experimental desktop control** at the bottom of Settings to show Desktop in the platform picker. Desktop runs use visual input by default: they capture the selected window and send physical pointer and keyboard input at the model's coordinates. Use `--desktop-input accessibility` for cursor-free CLI input instead. macOS requires Screen & System Audio Recording and Accessibility permissions for Droid or the terminal running the CLI. Windows targets must run at the same integrity level as Droid; a non-elevated Droid process cannot inject input into an elevated application.
+
 Config and precedence rules:
 - Use `--config <file>` to supply prompt-affecting settings for headless runs.
 - CLI flags override config file values.
@@ -293,7 +310,7 @@ HTML report (`--report`):
 - Writes a self-contained HTML report of the run — the same Run Details view the desktop app shows, with screenshots inlined — that you can share and open in any browser.
 - With no value it saves `<test>--<date>--<time>.html` (e.g. `login--2026-06-03--16-09-20.html`) in the current directory; pass `--report <file>` to pick a filename or `--report <dir>` to pick a folder.
 - In the desktop app the same report is available from a run's **Share → Download HTML report** button.
-- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 browser replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename. While rendering, a compact **Background tasks** card shows progress for **Generating test replay**; it can be minimized to a task-count card or canceled without opening the save dialog.
+- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 web and desktop replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename. While rendering, a compact **Background tasks** card shows progress for **Generating test replay**; it can be minimized to a task-count card or canceled without opening the save dialog.
 - Authenticated headless runs upload their HTML report and screenshots to Loadmill by default. The Loadmill run record also includes the resolved Droid version, model, provider, Agent Pace, Aim Assist, Context Optimization, and target environment. Use `--no-cloud-report` to disable artifact upload; the Droid run record is still reported.
 
 ```sh

package/THIRD_PARTY_NOTICES.md

@@ -0,0 +1,5 @@
+# Third-Party Notices
+
+Desktop control uses `@zavora-ai/computer-use-mcp` version 7.1.0.
+
+Copyright (c) Zavora AI contributors. Licensed under the MIT License. The dependency's complete license text is distributed with its npm package.

package/build/index.js

@@ -43,6 +43,7 @@ import { resolveRunPool } from "./src/device/cloud/loadmill-cloud/run-pool.js";
 import { ARTIFACT_READY_STATUSES } from "./src/device/cloud/loadmill-cloud/run-status.js";
 import { formatDuration } from "./src/core/report/run-log-model.js";
 import { createBrowserSessionTeardown, withLoadmillCloudSession } from "./src/cli/loadmill-cloud-session.js";
+import { createDesktopController, listDesktopApplications } from "./src/desktop/computer-use.js";
 dotenv.config();
 process.once("SIGTERM", async () => {
     process.exitCode = 143;
@@ -310,8 +311,20 @@ async function main() {
         console.log(`Run with: --device-name "<name>" --os-version "<osVersion>"`);
         process.exit(0);
     }
+    if (targetConfig.listDesktopApps) {
+        const applications = await listDesktopApplications({ excludePids: [process.pid] });
+        for (const application of applications) {
+            console.log(`${application.displayName} — ${application.bundleId} (pid ${application.pid})`);
+            for (const window of application.windows) {
+                console.log(`  ${window.windowId}: ${window.title || "Untitled"}`);
+            }
+        }
+        console.log(`\n${applications.length} desktop applications with visible windows.`);
+        process.exit(0);
+    }
     const isHeadlessInstructionsRun = Boolean(instructionsFile);
     const isBrowserTarget = targetConfig.target === "web";
+    const isDesktopTarget = targetConfig.target === "desktop";
     const isLambdaTestTarget = targetConfig.deviceSource === "lambdatest";
     const isAwsDeviceFarmTarget = targetConfig.deviceSource === "aws-device-farm";
     const isLoadmillCloudTarget = targetConfig.deviceSource === "loadmill-cloud";
@@ -326,6 +339,7 @@ async function main() {
     let loadmillCloudBrowserSessionConnected = false;
     let loadmillCloudBrowserTeardown = null;
     let loadmillCloudStatusLine = null;
+    let desktopController = null;
     const headlessDebug = createHeadlessDebugArtifacts({
         cwd: process.cwd(),
         enabled: isHeadlessInstructionsRun && debugMode
@@ -338,6 +352,9 @@ async function main() {
     if (isBrowserTarget && !isHeadlessInstructionsRun) {
         throw new Error("--target web is only supported with --instructions headless execution.");
     }
+    if (isDesktopTarget && !isHeadlessInstructionsRun) {
+        throw new Error("--target desktop is only supported with --instructions headless execution.");
+    }
     if (isCloudTarget && !isHeadlessInstructionsRun) {
         throw new Error(`--device-source ${targetConfig.deviceSource} is only supported with --instructions headless execution.`);
     }
@@ -398,7 +415,7 @@ async function main() {
         }
     }
     // If no device specified, show interactive selection menu
-    if (!isBrowserTarget && !isCloudTarget && !avdName && !platform) {
+    if (!isBrowserTarget && !isDesktopTarget && !isCloudTarget && !avdName && !platform) {
         const selection = await selectDevice();
         platform = selection.platform;
         avdName = selection.deviceName;
@@ -506,6 +523,24 @@ async function main() {
             console.log(`Starting URL: ${deviceInfo.currentUrl}`);
         }
     }
+    else if (isDesktopTarget) {
+        desktopController = await createDesktopController({
+            app: targetConfig.desktopApp,
+            windowId: targetConfig.desktopWindow,
+            inputMode: targetConfig.desktopInput,
+            excludePids: [process.pid],
+        });
+        targetId = desktopController.deviceId;
+        deviceInfo = desktopController.deviceInfo;
+        console.log(`Attached to desktop window: ${deviceInfo.device_name}`);
+        console.log(`Model sees window: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+        console.log(`Desktop input: ${deviceInfo.desktopInputMode === "physical" ? "physical OS pointer" : "cursor-free accessibility"}`);
+        process.once("SIGINT", async () => {
+            console.log("\nStopping desktop run...");
+            await desktopController?.close().catch(() => { });
+            process.exit(130);
+        });
+    }
     else if (isLambdaTestTarget) {
         lambdaTestCredentials = resolveLambdaTestCredentials(process.env);
         const lambdaTestBuildName = createLambdaTestCliBuildName();
@@ -787,8 +822,10 @@ async function main() {
                 recordScreenshots,
                 screenshotDir,
                 isBrowserTarget,
+                isDesktopTarget,
+                desktopController,
                 isCloudTarget,
-                runTarget: { isBrowserTarget, isCloudTarget },
+                runTarget: { isBrowserTarget, isDesktopTarget, isCloudTarget },
                 report: reportConfig
             });
             exitCode = result.exitCode;
@@ -845,6 +882,10 @@ async function main() {
                     console.log(`Done in ${formatDuration(Date.now() - processStartTime)}.`);
                 }
             }
+            if (desktopController) {
+                await desktopController.close();
+                desktopController = null;
+            }
         }
         if (exitCode !== null) {
             process.exit(exitCode);

package/build/src/cli/headless-runner.js

@@ -83,9 +83,10 @@ function emitCliAppContextDebug(runId, appContext, appContextText) {
         });
     }
 }
-export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionPrompt, screenshotRecorder, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, runTarget = null, onCuaResponse = null, secrets = null }) {
-    const resolvedRunTarget = runTarget ?? { isBrowserTarget, isCloudTarget };
+export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionPrompt, screenshotRecorder, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget, desktopController, isCloudTarget, runTarget = null, onCuaResponse = null, secrets = null }) {
+    const resolvedRunTarget = runTarget ?? { isBrowserTarget, isDesktopTarget, isCloudTarget };
     const isBrowserRunTarget = Boolean(resolvedRunTarget.isBrowserTarget);
+    const isDesktopRunTarget = Boolean(resolvedRunTarget.isDesktopTarget);
     const isCloudRunTarget = Boolean(resolvedRunTarget.isCloudTarget);
     const sendCUARequest = async (requestArgs) => {
         const response = await runtimeContext.openAi.sendCUARequest({
@@ -101,7 +102,7 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
         toolCapabilities: {
             assert: true,
             navigate: isBrowserRunTarget,
-            closeKeyboard: !isBrowserRunTarget,
+            closeKeyboard: !isBrowserRunTarget && !isDesktopRunTarget,
             loadmill: true,
         },
         secretsByKey: secrets?.secretsByKey || {},
@@ -130,6 +131,11 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
         engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
         engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
     }
+    else if (isDesktopRunTarget) {
+        engineOptions.getScreenshotAsBase64 = desktopController.getScreenshotAsBase64;
+        engineOptions.handleModelAction = desktopController.handleModelAction;
+        engineOptions.getCurrentPlatform = () => "desktop";
+    }
     if (isCloudRunTarget) {
         if (isBrowserRunTarget) {
             engineOptions.connectToBrowser = createRefuseReconnectConnectToBrowser();
@@ -142,7 +148,7 @@ export function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, exe
     }
     return engineOptions;
 }
-async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, runTarget = null, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, runConfiguration = {}, cloudUploadSession = null, uploadManager = droidReportUploadManager, }) {
+async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget, desktopController, isCloudTarget, runTarget = null, secrets = null, report = null, isBatchRun = false, reportToLoadmill = true, runConfiguration = {}, cloudUploadSession = null, uploadManager = droidReportUploadManager, }) {
     let executionMode = null;
     let loadmillSessionUser = null;
     let runUsageTotals = setDroidCuaRunUsageModel(createDroidCuaRunUsageTotals(), runtimeContext.config.cuaModel);
@@ -218,6 +224,8 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             recordScreenshots,
             screenshotDir,
             isBrowserTarget,
+            isDesktopTarget,
+            desktopController,
             isCloudTarget,
             runTarget,
             onCuaResponse: (response) => {
@@ -428,8 +436,8 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         throw error;
     }
 }
-export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isCloudTarget, runTarget = null, report = null, droidVersion = getDroidVersion(), runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, }) {
-    const resolvedRunTarget = runTarget ?? { isBrowserTarget, isCloudTarget };
+export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, secrets = null, recordScreenshots, screenshotDir, isBrowserTarget, isDesktopTarget = false, desktopController = null, isCloudTarget, runTarget = null, report = null, droidVersion = getDroidVersion(), runHeadlessTestImpl = runHeadlessTest, uploadManager = droidReportUploadManager, }) {
+    const resolvedRunTarget = runTarget ?? { isBrowserTarget, isDesktopTarget, isCloudTarget };
     const runConfiguration = buildDroidRunConfiguration({
         droidVersion,
         runtimeConfig: runtimeContext.config,
@@ -489,6 +497,8 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
                 recordScreenshots,
                 screenshotDir: testScreenshotDir,
                 isBrowserTarget: resolvedRunTarget.isBrowserTarget,
+                isDesktopTarget: resolvedRunTarget.isDesktopTarget,
+                desktopController,
                 isCloudTarget: resolvedRunTarget.isCloudTarget,
                 runTarget: resolvedRunTarget,
                 report,

package/build/src/cli/headless-target.js

@@ -4,13 +4,13 @@ function normalizeTarget(value) {
         return "mobile";
     }
     if (typeof value !== "string") {
-        throw new Error("--target must be one of: mobile, web.");
+        throw new Error("--target must be one of: mobile, web, desktop.");
     }
     const normalized = value.trim().toLowerCase();
-    if (normalized === "mobile" || normalized === "web") {
+    if (normalized === "mobile" || normalized === "web" || normalized === "desktop") {
         return normalized;
     }
-    throw new Error("--target must be one of: mobile, web.");
+    throw new Error("--target must be one of: mobile, web, desktop.");
 }
 function normalizeSessionMode(value) {
     if (value == null || value === "") {
@@ -251,6 +251,53 @@ export function resolveHeadlessTarget(args = {}) {
             listDevices: false,
         };
     }
+    if (target === "desktop") {
+        if (avdName || platform || args.browser != null || args["session-mode"] != null) {
+            throw new Error("--avd, --platform, --browser, and --session-mode cannot be used with --target desktop.");
+        }
+        if (deviceSource !== "local") {
+            throw new Error("--device-source can only be used with --target mobile.");
+        }
+        assertNoCloudFlags(args);
+        if (args["list-devices"] != null) {
+            throw new Error("--list-devices can only be used with --target mobile.");
+        }
+        const listDesktopApps = args["list-desktop-apps"] != null;
+        const desktopApp = hasNonEmptyString(args["desktop-app"]) ? args["desktop-app"].trim() : null;
+        const rawWindow = readNonEmptyCliValue(args["desktop-window"]);
+        const desktopWindow = rawWindow == null ? null : Number(rawWindow);
+        const desktopInput = hasNonEmptyString(args["desktop-input"]) ? args["desktop-input"].trim().toLowerCase() : "physical";
+        if (!listDesktopApps && !desktopApp) {
+            throw new Error("--desktop-app is required with --target desktop.");
+        }
+        if (rawWindow != null && (!Number.isInteger(desktopWindow) || desktopWindow < 0)) {
+            throw new Error("--desktop-window must be a non-negative integer window ID.");
+        }
+        if (desktopInput !== "accessibility" && desktopInput !== "physical") {
+            throw new Error('--desktop-input must be "accessibility" or "physical".');
+        }
+        return {
+            target,
+            sessionMode: null,
+            browserFamily: null,
+            deviceSource: "local",
+            avdName: null,
+            platform: null,
+            deviceName: null,
+            osVersion: null,
+            appPath: null,
+            cloudSessionId: null,
+            artifacts: null,
+            listDevices: false,
+            listDesktopApps,
+            desktopApp,
+            desktopWindow,
+            desktopInput,
+        };
+    }
+    if (args["desktop-app"] != null || args["desktop-window"] != null || args["desktop-input"] != null || args["list-desktop-apps"] != null) {
+        throw new Error("--desktop-app, --desktop-window, --desktop-input, and --list-desktop-apps can only be used with --target desktop.");
+    }
     if (args["session-mode"] != null) {
         throw new Error("--session-mode can only be used with --target web.");
     }

package/build/src/commands/help.js

@@ -19,7 +19,11 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  droid-cua --device-source lambdatest --instructions <file> [options]' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Options:' });
-    addOutput({ type: 'info', text: '  --target <kind>        Target kind: mobile (default) or web' });
+    addOutput({ type: 'info', text: '  --target <kind>        Target kind: mobile (default), web, or desktop' });
+    addOutput({ type: 'info', text: '  --desktop-app <id>     Running desktop app ID or exact name' });
+    addOutput({ type: 'info', text: '  --desktop-window <id>  Window ID when the app has multiple windows' });
+    addOutput({ type: 'info', text: '  --desktop-input <mode> Physical pointer (default) or cursor-free accessibility' });
+    addOutput({ type: 'info', text: '  --list-desktop-apps    List running desktop apps and windows' });
     addOutput({ type: 'info', text: '  --avd <name>           Device name (Android device ID/serial or iOS Simulator)' });
     addOutput({ type: 'info', text: '  --platform <platform>  Force platform: android or ios' });
     addOutput({ type: 'info', text: '  --device-source <src>  Mobile source: local (default) or lambdatest' });

package/build/src/core/session.js

@@ -13,7 +13,11 @@ export class Session {
         this.browserName = targetInfo?.browserName || null;
         this.currentUrl = targetInfo?.currentUrl || null;
         this.sessionMode = targetInfo?.sessionMode || null;
-        this.targetKind = targetInfo?.platform === "browser" ? "browser" : "mobile";
+        this.targetKind = targetInfo?.platform === "browser"
+            ? "browser"
+            : targetInfo?.platform === "desktop"
+                ? "desktop"
+                : "mobile";
         this.messages = [];
         this.previousResponseId = null;
         this.transcript = [];

package/build/src/core/test-data-import.js

@@ -1,13 +1,13 @@
 import path from "path";
 import { resolveEffectiveHelperModel } from "./runtime-config.js";
 import { buildTestDataImportMessages } from "../prompts/test-data-import.js";
-export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt"];
+export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt", ".json", ".xml", ".md"];
 export function isSupportedTestDataImportPath(filePath) {
     return TEST_DATA_IMPORT_EXTENSIONS.includes(path.extname(String(filePath ?? "")).toLowerCase());
 }
 export function assertSupportedTestDataImportPath(filePath) {
     if (!isSupportedTestDataImportPath(filePath)) {
-        throw new Error("Test data import supports CSV and TXT files only.");
+        throw new Error("Choose a CSV, TXT, JSON, XML, or Markdown file.");
     }
 }
 function parseTestDataImportContent(content) {

package/build/src/desktop/computer-use.js

@@ -0,0 +1,432 @@
+import { buildActionEventMeta } from "../device/action-event-meta.js";
+import { buildTypeActionPayload, buildTypeActionText, } from "../device/sensitive-action.js";
+import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+const SCREENSHOT_WIDTH = 1440;
+const ACCESSIBILITY_HIT_TOLERANCE = 12;
+const MACOS_CAPTURE_INSETS = { left: 50, right: 50, top: 38, bottom: 62 };
+const SUPPORTED_PLATFORMS = new Set(["darwin", "win32"]);
+const INPUT_MODES = new Set(["accessibility", "physical"]);
+function parseJsonResult(result, label) {
+    if (result?.isError) {
+        throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
+    }
+    if (result?.structuredContent) {
+        return result.structuredContent;
+    }
+    const text = result?.content?.find((item) => item.type === "text")?.text;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`${label} returned an invalid response.`);
+    }
+}
+function assertResult(result, label) {
+    if (result?.isError) {
+        throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
+    }
+}
+function sameBounds(left, right) {
+    return left && right && left.x === right.x && left.y === right.y && left.width === right.width && left.height === right.height;
+}
+function getScreenshotContentBounds(width, height, windowBounds, platform) {
+    const fullScreenshot = { x: 0, y: 0, width, height };
+    if (platform !== "darwin")
+        return fullScreenshot;
+    const { left, right, top, bottom } = MACOS_CAPTURE_INSETS;
+    const scaleX = width / (windowBounds.width + left + right);
+    const scaleY = height / (windowBounds.height + top + bottom);
+    if (Math.abs(scaleX - scaleY) / Math.max(scaleX, scaleY) > 0.02)
+        return fullScreenshot;
+    // ponytail: computer-use-mcp v7 omits capture bounds; remove this calibration when upstream exposes them.
+    return {
+        x: left * scaleX,
+        y: top * scaleY,
+        width: windowBounds.width * scaleX,
+        height: windowBounds.height * scaleY,
+    };
+}
+function formatPermissionError(error, platform) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (platform === "darwin" && /permission|not authorized|screencapture|screen recording|cgwindow|axerror/i.test(message)) {
+        return `${message} Grant Screen & System Audio Recording and Accessibility access to Droid (or your terminal for CLI runs) in System Settings > Privacy & Security, then restart it.`;
+    }
+    if (platform === "win32" && /access denied|elevat|integrity level|could not focus/i.test(message)) {
+        return `${message} Run Droid in an interactive desktop session and at the same integrity level as the target application; elevated Windows applications cannot be controlled by a non-elevated Droid process.`;
+    }
+    return message;
+}
+async function createLibraryClient() {
+    const [{ createComputerUseServer }, { connectInProcess }] = await Promise.all([
+        import("@zavora-ai/computer-use-mcp"),
+        import("@zavora-ai/computer-use-mcp/client"),
+    ]);
+    return connectInProcess(createComputerUseServer({ profile: "ax", vision: true }));
+}
+async function loadAccessibilityDriver() {
+    const { loadNative } = await import("@zavora-ai/computer-use-mcp/host-native");
+    return loadNative();
+}
+function walkAccessibilityTree(root, depth = 0, elements = []) {
+    if (!root || typeof root !== "object")
+        return elements;
+    elements.push({ ...root, depth });
+    for (const child of Array.isArray(root.children) ? root.children : []) {
+        walkAccessibilityTree(child, depth + 1, elements);
+    }
+    return elements;
+}
+function distanceToBounds(bounds, x, y) {
+    if (!bounds || bounds.width <= 0 || bounds.height <= 0)
+        return Infinity;
+    const dx = Math.max(bounds.x - x, 0, x - bounds.x - bounds.width);
+    const dy = Math.max(bounds.y - y, 0, y - bounds.y - bounds.height);
+    return Math.hypot(dx, dy);
+}
+function actionFor(element, actionType, direction = null) {
+    const actions = Array.isArray(element.actions) ? element.actions : [];
+    if (actionType === "click")
+        return actions.find((action) => /press|invoke|click/i.test(action));
+    if (actionType === "double_click")
+        return actions.find((action) => /open|double.?click/i.test(action));
+    if (actionType === "scroll") {
+        const expected = direction === "up" ? /scroll.?up/i
+            : direction === "down" ? /scroll.?down/i
+                : direction === "left" ? /scroll.?left/i
+                    : /scroll.?right/i;
+        return actions.find((action) => expected.test(action));
+    }
+    return null;
+}
+function chooseAccessibilityElement(root, x, y, actionType, direction = null) {
+    const elements = walkAccessibilityTree(root);
+    const candidates = elements
+        .map((element) => ({ element, action: actionFor(element, actionType, direction), distance: distanceToBounds(element.bounds, x, y) }))
+        .filter(({ element, action, distance }) => action && element.role && String(element.label || "").trim() && distance <= ACCESSIBILITY_HIT_TOLERANCE)
+        .sort((left, right) => {
+        const leftArea = left.element.bounds.width * left.element.bounds.height;
+        const rightArea = right.element.bounds.width * right.element.bounds.height;
+        return left.distance - right.distance || leftArea - rightArea || right.element.depth - left.element.depth;
+    });
+    const selected = candidates[0];
+    if (!selected)
+        return null;
+    if (selected.distance > 0 && candidates[1]?.distance === selected.distance)
+        return null;
+    const duplicates = elements.filter((element) => element.role === selected.element.role && element.label === selected.element.label);
+    return duplicates.length === 1 ? selected : null;
+}
+function cursorFreeError(actionType) {
+    return `Cursor-free desktop ${actionType.replaceAll("_", " ")} is unavailable at this location because the application did not expose a unique accessibility action. No mouse input was sent. Re-run with --desktop-input physical to allow OS pointer control.`;
+}
+function cursorFreeKeyboardError(actionType) {
+    return `Cursor-free desktop ${actionType.replaceAll("_", " ")} is unavailable because keyboard input requires application focus. No keyboard input was sent. Re-run with --desktop-input physical to allow focused OS input.`;
+}
+function readApplications(result) {
+    const data = parseJsonResult(result, "Application discovery");
+    return Array.isArray(data) ? data : [];
+}
+async function discoverApplications(client) {
+    let applications = readApplications(await client.listRunningApps());
+    if (applications.length === 0) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        applications = readApplications(await client.listRunningApps());
+    }
+    return applications;
+}
+function readWindows(result) {
+    const data = parseJsonResult(result, "Window discovery");
+    return Array.isArray(data) ? data : Array.isArray(data.windows) ? data.windows : [];
+}
+function chooseApplication(apps, requestedApp) {
+    const needle = String(requestedApp || "").trim().toLowerCase();
+    const matches = apps.filter((app) => app.bundleId?.toLowerCase() === needle || app.displayName?.trim().toLowerCase() === needle);
+    if (matches.length === 1)
+        return matches[0];
+    if (matches.length > 1) {
+        throw new Error(`Desktop application "${requestedApp}" is ambiguous. Use its application ID instead.`);
+    }
+    throw new Error(`Desktop application "${requestedApp}" is not running. Use --list-desktop-apps to see available applications.`);
+}
+function chooseWindow(windows, requestedWindowId) {
+    const visible = windows.filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+    if (requestedWindowId != null) {
+        const selected = visible.find((window) => window.windowId === requestedWindowId);
+        if (!selected)
+            throw new Error(`Desktop window ${requestedWindowId} was not found for the selected application.`);
+        return selected;
+    }
+    const focused = visible.filter((window) => window.isFocused);
+    if (focused.length === 1)
+        return focused[0];
+    if (visible.length === 1)
+        return visible[0];
+    if (visible.length === 0)
+        throw new Error("The selected desktop application has no visible windows.");
+    const choices = visible.map((window) => `${window.windowId}: ${window.title || "Untitled"}`).join("\n");
+    throw new Error(`The selected desktop application has multiple windows. Choose one with --desktop-window <id>:\n${choices}`);
+}
+function normalizeShortcut(keys, platform) {
+    if (!Array.isArray(keys) || keys.length === 0)
+        throw new Error("Keypress action is missing keys.");
+    const aliases = {
+        CTRL: "control",
+        CONTROL: "control",
+        CMD: "command",
+        COMMAND: "command",
+        META: platform === "darwin" ? "command" : "win",
+        OPTION: "alt",
+        ALT: "alt",
+        ESC: "escape",
+        RETURN: "enter",
+    };
+    return keys.map((key) => aliases[String(key).toUpperCase()] || String(key).toLowerCase()).join("+");
+}
+/**
+ * @param {{ app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", client?: any, accessibility?: any }} options
+ */
+export async function createDesktopController({ app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", client: injectedClient = null, accessibility: injectedAccessibility = null, } = {}) {
+    if (!SUPPORTED_PLATFORMS.has(platform)) {
+        throw new Error("Desktop control is currently supported only on macOS and Windows.");
+    }
+    if (!requestedApp)
+        throw new Error("--desktop-app is required with --target desktop.");
+    if (!INPUT_MODES.has(inputMode))
+        throw new Error('Desktop input mode must be "accessibility" or "physical".');
+    const client = injectedClient || await createLibraryClient();
+    try {
+        const apps = (await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid));
+        const application = chooseApplication(apps, requestedApp);
+        const windows = readWindows(await client.listWindows(application.bundleId));
+        let targetWindow = chooseWindow(windows, requestedWindowId == null ? null : Number(requestedWindowId));
+        let capture = null;
+        let deviceInfo = null;
+        let accessibility = injectedAccessibility;
+        async function getWindow() {
+            return parseJsonResult(await client.getWindow(targetWindow.windowId), "Window lookup");
+        }
+        async function screenshot() {
+            let window;
+            let result;
+            try {
+                const currentWindows = readWindows(await client.listWindows(application.bundleId));
+                const focusedWindow = currentWindows.find((candidate) => candidate.isFocused && candidate.isOnScreen);
+                if (focusedWindow)
+                    targetWindow = focusedWindow;
+                window = await getWindow();
+                if (!window.isOnScreen)
+                    throw new Error("The selected desktop window is no longer visible.");
+                result = await client.screenshot({
+                    width: SCREENSHOT_WIDTH,
+                    quality: 0,
+                    target_window_id: targetWindow.windowId,
+                });
+                assertResult(result, "Desktop screenshot");
+            }
+            catch (error) {
+                throw new Error(formatPermissionError(error, platform));
+            }
+            const image = result.content?.find((item) => item.type === "image");
+            const sizeText = result.content?.find((item) => item.type === "text")?.text || "";
+            const size = /^(\d+)x(\d+)$/.exec(sizeText.trim());
+            if (!image?.data || !size)
+                throw new Error("Desktop screenshot returned no image or dimensions.");
+            capture = {
+                bounds: { ...window.bounds },
+                width: Number(size[1]),
+                height: Number(size[2]),
+            };
+            capture.contentBounds = getScreenshotContentBounds(capture.width, capture.height, window.bounds, platform);
+            if (deviceInfo) {
+                deviceInfo.device_width = window.bounds.width;
+                deviceInfo.device_height = window.bounds.height;
+                deviceInfo.scaled_width = capture.width;
+                deviceInfo.scaled_height = capture.height;
+                deviceInfo.scale = capture.width / window.bounds.width;
+                deviceInfo.desktopWindowId = targetWindow.windowId;
+            }
+            return image.data;
+        }
+        function scaledPoint(x, y) {
+            if (!capture)
+                throw new Error("Capture the desktop window before executing an action.");
+            const content = capture.contentBounds;
+            return [
+                Math.round(capture.bounds.x + ((x - content.x) * capture.bounds.width) / content.width),
+                Math.round(capture.bounds.y + ((y - content.y) * capture.bounds.height) / content.height),
+            ];
+        }
+        async function prepareAction(requireFocus) {
+            if (!capture)
+                throw new Error("Capture the desktop window before executing an action.");
+            const before = await getWindow();
+            if (!sameBounds(before.bounds, capture.bounds)) {
+                throw new Error("The desktop window moved or resized after the screenshot. A fresh screenshot is required before acting.");
+            }
+            if (!requireFocus)
+                return;
+            const activation = parseJsonResult(await client.activateWindow(targetWindow.windowId, 2000), "Window activation");
+            const focused = await getWindow();
+            if (!activation.activated || !focused.isFocused) {
+                throw new Error("Droid could not focus the selected desktop window, so no input was sent.");
+            }
+        }
+        const targetOptions = () => ({ targetWindowId: targetWindow.windowId, focusStrategy: "strict" });
+        async function performCursorFreeAction(actionType, x, y, direction = null) {
+            const tree = parseJsonResult(await client.getUiTree(targetWindow.windowId, 20), "Accessibility tree");
+            const selected = chooseAccessibilityElement(tree, x, y, actionType, direction);
+            if (!selected)
+                throw new Error(cursorFreeError(actionType));
+            accessibility ||= await loadAccessibilityDriver();
+            const result = accessibility.performAction(targetWindow.windowId, selected.element.role, selected.element.label, selected.action);
+            if (!result?.performed)
+                throw new Error(`${cursorFreeError(actionType)}${result?.reason ? ` Reason: ${result.reason}.` : ""}`);
+        }
+        async function handleAction(_deviceId, action, _scale = 1, context = null) {
+            const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+            const meta = (payload = {}) => buildActionEventMeta({ context, action, platform: "desktop", payload });
+            try {
+                if (context?.shouldStop?.())
+                    return { success: false, error: "Stopped by user" };
+                if (inputMode === "accessibility" && (action.type === "type" || action.type === "keypress")) {
+                    throw new Error(cursorFreeKeyboardError(action.type));
+                }
+                const requiresFocus = inputMode === "physical";
+                if (action.type !== "wait")
+                    await prepareAction(requiresFocus);
+                switch (action.type) {
+                    case "click":
+                    case "double_click": {
+                        const [x, y] = scaledPoint(action.x, action.y);
+                        addOutput({ type: "action", text: action.type === "click" ? "Click" : `Double-clicking at (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
+                        if (inputMode === "accessibility") {
+                            await performCursorFreeAction(action.type, x, y);
+                        }
+                        else {
+                            const result = action.type === "click"
+                                ? await client.click(x, y, application.bundleId, targetOptions())
+                                : await client.doubleClick(x, y, application.bundleId, targetOptions());
+                            assertResult(result, "Desktop click");
+                        }
+                        break;
+                    }
+                    case "move": {
+                        const [x, y] = scaledPoint(action.x, action.y);
+                        addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
+                        if (inputMode === "accessibility")
+                            throw new Error(cursorFreeError(action.type));
+                        assertResult(await client.moveMouse(x, y, application.bundleId, targetOptions()), "Desktop mouse move");
+                        break;
+                    }
+                    case "drag": {
+                        if (!Array.isArray(action.path) || action.path.length < 2)
+                            throw new Error("Drag action requires a path with at least two points.");
+                        const from = scaledPoint(action.path[0].x, action.path[0].y);
+                        const last = action.path[action.path.length - 1];
+                        const to = scaledPoint(last.x, last.y);
+                        addOutput({ type: "action", text: `Dragging from (${from[0]}, ${from[1]}) to (${to[0]}, ${to[1]})`, ...meta({ pathStart: { x: from[0], y: from[1] }, pathEnd: { x: to[0], y: to[1] }, unit: "px" }) });
+                        if (inputMode === "accessibility")
+                            throw new Error(cursorFreeError(action.type));
+                        assertResult(await client.drag(to, from, application.bundleId, targetOptions()), "Desktop drag");
+                        break;
+                    }
+                    case "scroll": {
+                        const [x, y] = scaledPoint(action.x ?? capture.width / 2, action.y ?? capture.height / 2);
+                        const scrollX = Number(action.scroll_x) || 0;
+                        const scrollY = Number(action.scroll_y) || 0;
+                        const horizontal = Math.abs(scrollX) > Math.abs(scrollY);
+                        const direction = horizontal ? (scrollX > 0 ? "right" : "left") : (scrollY > 0 ? "down" : "up");
+                        const amount = Math.max(1, Math.min(10, Math.round(Math.abs(horizontal ? scrollX : scrollY) / 100)));
+                        addOutput({ type: "action", text: `Scrolling ${direction}`, ...meta({ scrollX, scrollY, x, y, unit: "px" }) });
+                        if (inputMode === "accessibility") {
+                            await performCursorFreeAction(action.type, x, y, direction);
+                        }
+                        else {
+                            assertResult(await client.scroll(x, y, direction, amount, application.bundleId, targetOptions()), "Desktop scroll");
+                        }
+                        break;
+                    }
+                    case "type": {
+                        const text = typeof action.text === "string" ? action.text : "";
+                        addOutput({ type: "action", text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
+                        assertResult(await client.type(text, application.bundleId, targetOptions()), "Desktop typing");
+                        break;
+                    }
+                    case "keypress": {
+                        const shortcut = normalizeShortcut(action.keys, platform);
+                        addOutput({ type: "action", text: `Pressing key: ${shortcut}`, ...meta({ keys: action.keys, shortcut }) });
+                        assertResult(await client.key(shortcut, application.bundleId, targetOptions()), "Desktop keypress");
+                        break;
+                    }
+                    case "wait": {
+                        addOutput({ type: "action", text: "Waiting...", ...meta() });
+                        const timeoutMs = Number.isFinite(action.ms) && action.ms >= 0 ? Math.round(action.ms) : getConfiguredStepDelayMs();
+                        await new Promise((resolve) => setTimeout(resolve, timeoutMs));
+                        break;
+                    }
+                    default:
+                        addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+                }
+                return { success: true };
+            }
+            catch (error) {
+                const message = formatPermissionError(error, platform);
+                addOutput({ type: "error", text: message, eventType: "error", actionType: action?.type, payload: { message, platform: "desktop" } });
+                return { success: false, error: message };
+            }
+        }
+        await screenshot();
+        deviceInfo = {
+            platform: "desktop",
+            source: "local",
+            device_name: `${application.displayName} — ${targetWindow.title || "Untitled"}`,
+            device_width: targetWindow.bounds.width,
+            device_height: targetWindow.bounds.height,
+            scaled_width: capture.width,
+            scaled_height: capture.height,
+            scale: capture.width / targetWindow.bounds.width,
+            desktopAppId: application.bundleId,
+            desktopWindowId: targetWindow.windowId,
+            desktopInputMode: inputMode,
+        };
+        return {
+            application,
+            window: targetWindow,
+            deviceId: `desktop:${targetWindow.windowId}`,
+            deviceInfo,
+            getScreenshotAsBase64: screenshot,
+            handleModelAction: handleAction,
+            close: () => client.close(),
+        };
+    }
+    catch (error) {
+        await client.close().catch(() => { });
+        throw error;
+    }
+}
+/**
+ * @param {{ excludePids?: number[], platform?: string, client?: any }} options
+ */
+export async function listDesktopApplications({ excludePids = [], platform = process.platform, client: injectedClient = null } = {}) {
+    if (!SUPPORTED_PLATFORMS.has(platform)) {
+        throw new Error("Desktop control is currently supported only on macOS and Windows.");
+    }
+    const client = injectedClient || await createLibraryClient();
+    try {
+        const apps = (await discoverApplications(client))
+            .filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid))
+            .sort((left, right) => left.displayName.localeCompare(right.displayName));
+        const entries = [];
+        for (const app of apps) {
+            const windows = readWindows(await client.listWindows(app.bundleId))
+                .filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+            if (windows.length > 0)
+                entries.push({ ...app, windows });
+        }
+        return entries;
+    }
+    finally {
+        if (!injectedClient)
+            await client.close();
+    }
+}

package/build/src/prompts/base.js

@@ -30,6 +30,9 @@ ${customSection}
 function isBrowserTarget(deviceInfo = {}) {
     return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
 }
+function isDesktopTarget(deviceInfo = {}) {
+    return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "desktop";
+}
 function isCloudIosTarget(deviceInfo = {}) {
     const platform = deviceInfo.platform?.trim().toLowerCase();
     const source = deviceInfo.source?.trim().toLowerCase() || "local";
@@ -42,6 +45,9 @@ function describeControlledTarget(deviceInfo = {}) {
     if (platform === "browser") {
         return browserName ? `a browser session (${browserName})` : "a browser session";
     }
+    if (platform === "desktop") {
+        return deviceName ? `a desktop application (${deviceName})` : "a desktop application";
+    }
     if (platform === "ios") {
         const target = isCloudIosTarget(deviceInfo) ? "an iOS device" : "an iOS simulator";
         return deviceName ? `${target} (${deviceName})` : target;
@@ -248,9 +254,37 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
   - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
   `;
 }
+function buildDesktopBaseSystemPrompt(deviceInfo, options = {}) {
+    let browserPrompt = buildBrowserBaseSystemPrompt(deviceInfo, options)
+        .replace("Follow the user's instructions to interact with the browser.", "Follow the user's instructions to interact with the selected application.")
+        .replace("visible browser window", "selected application window")
+        .replace("CRITICAL - Desktop Browser Input Constraints:", "CRITICAL - Desktop Input Constraints:")
+        .replace("This is a desktop browser environment.", "This is a desktop application environment.");
+    if (deviceInfo.desktopInputMode !== "physical") {
+        browserPrompt = browserPrompt
+            .replace("Available actions: click, scroll, type, keypress, wait, screenshot.", "Available actions: click, scroll, wait, screenshot.")
+            .replace("Standard desktop keys and modifier chords are allowed when they are clearly useful.", "Keyboard input is unavailable because it would focus the application.")
+            .replace("Prefer clicking visible controls over keyboard shortcuts when both are equally reliable.", "Use visible application controls for every interaction.")
+            .replace("Use keypress chords only when they directly help the task, such as Enter to submit, Escape to dismiss a modal, or Control/Meta shortcuts for focused text fields.", "Use visible application controls for submit, dismiss, and correction actions.");
+        browserPrompt = browserPrompt.replace("After EVERY action (click, type, keypress, scroll)", "After EVERY action (click, scroll)");
+    }
+    const notesIndex = browserPrompt.indexOf("  Browser-Specific Notes:");
+    const withoutBrowserNotes = notesIndex === -1 ? browserPrompt : browserPrompt.slice(0, notesIndex);
+    const inputNote = deviceInfo.desktopInputMode === "physical"
+        ? "Droid may use the physical OS pointer for this run"
+        : "Cursor-free mode never focuses the application: use only clicks and scrolling on controls with matching accessibility actions; never emit type, keypress, move, hover, or drag";
+    return `${withoutBrowserNotes}  Desktop-Specific Notes:
+  - Droid is controlling only the selected application window, not the whole display
+  - ${inputNote}
+  - Do not use shortcuts that switch applications, virtual desktops, or operating-system context
+  - Dialogs owned by the selected application may be controlled when they become its active window
+  `;
+}
 function buildBaseSystemPrompt(deviceInfo, options = {}) {
-    return isBrowserTarget(deviceInfo)
-        ? buildBrowserBaseSystemPrompt(deviceInfo, options)
-        : buildMobileBaseSystemPrompt(deviceInfo, options);
+    if (isBrowserTarget(deviceInfo))
+        return buildBrowserBaseSystemPrompt(deviceInfo, options);
+    if (isDesktopTarget(deviceInfo))
+        return buildDesktopBaseSystemPrompt(deviceInfo, options);
+    return buildMobileBaseSystemPrompt(deviceInfo, options);
 }
-export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, };
+export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, isDesktopTarget, };

package/build/src/prompts/test-data-import.js

@@ -17,8 +17,8 @@ export function buildTestDataImportMessages({ currentContent = "", importedFileN
                 "- Do not invent values that are not present in the current file, imported file, or user instructions.",
                 "- If currentContent is template-only, replace it with real imported data and do not keep commented template examples.",
                 "- If currentContent is not template-only, preserve its useful structure and add or update entities within it.",
-                "- For matrix-like CSV rows, prefer splitting repeated dimensions into separate entities when the structure is clear.",
-                "- If rows are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
+                "- For tabular or nested data, preserve meaningful groups and relationships. Split repeated dimensions into separate entities when the structure is clear.",
+                "- If records are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
                 "- User instructions can guide structure and naming, but content must remain concrete data, not test instructions.",
                 "- Output markdown only inside the JSON content string; do not use code fences.",
             ].join("\n")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.34.0",
+  "version": "2.35.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -12,6 +12,7 @@
     "bin/**/*",
     "skill/**/*",
     "README.md",
+    "THIRD_PARTY_NOTICES.md",
     "LICENSE"
   ],
   "scripts": {
@@ -63,6 +64,7 @@
   "license": "UNLICENSED",
   "dependencies": {
     "@aws-sdk/client-device-farm": "^3.1065.0",
+    "@zavora-ai/computer-use-mcp": "7.1.0",
     "dotenv": "^16.4.7",
     "ink": "^4.4.1",
     "ink-spinner": "^5.0.0",
@@ -78,6 +80,6 @@
     "typescript": "^5.7.3"
   },
   "engines": {
-    "node": ">=18.17.0"
+    "node": ">=20.0.0"
   }
 }