@loadmill/droid-cua 2.34.0 → 2.36.0

package/build/src/cli/headless-target.js

@@ -4,13 +4,13 @@ function normalizeTarget(value) {
         return "mobile";
     }
     if (typeof value !== "string") {
-        throw new Error("--target must be one of: mobile, web.");
+        throw new Error("--target must be one of: mobile, web, desktop.");
     }
     const normalized = value.trim().toLowerCase();
-    if (normalized === "mobile" || normalized === "web") {
+    if (normalized === "mobile" || normalized === "web" || normalized === "desktop") {
         return normalized;
     }
-    throw new Error("--target must be one of: mobile, web.");
+    throw new Error("--target must be one of: mobile, web, desktop.");
 }
 function normalizeSessionMode(value) {
     if (value == null || value === "") {
@@ -251,6 +251,53 @@ export function resolveHeadlessTarget(args = {}) {
             listDevices: false,
         };
     }
+    if (target === "desktop") {
+        if (avdName || platform || args.browser != null || args["session-mode"] != null) {
+            throw new Error("--avd, --platform, --browser, and --session-mode cannot be used with --target desktop.");
+        }
+        if (deviceSource !== "local") {
+            throw new Error("--device-source can only be used with --target mobile.");
+        }
+        assertNoCloudFlags(args);
+        if (args["list-devices"] != null) {
+            throw new Error("--list-devices can only be used with --target mobile.");
+        }
+        const listDesktopApps = args["list-desktop-apps"] != null;
+        const desktopApp = hasNonEmptyString(args["desktop-app"]) ? args["desktop-app"].trim() : null;
+        const rawWindow = readNonEmptyCliValue(args["desktop-window"]);
+        const desktopWindow = rawWindow == null ? null : Number(rawWindow);
+        const desktopInput = hasNonEmptyString(args["desktop-input"]) ? args["desktop-input"].trim().toLowerCase() : "physical";
+        if (!listDesktopApps && !desktopApp) {
+            throw new Error("--desktop-app is required with --target desktop.");
+        }
+        if (rawWindow != null && (!Number.isInteger(desktopWindow) || desktopWindow < 0)) {
+            throw new Error("--desktop-window must be a non-negative integer window ID.");
+        }
+        if (desktopInput !== "accessibility" && desktopInput !== "physical") {
+            throw new Error('--desktop-input must be "accessibility" or "physical".');
+        }
+        return {
+            target,
+            sessionMode: null,
+            browserFamily: null,
+            deviceSource: "local",
+            avdName: null,
+            platform: null,
+            deviceName: null,
+            osVersion: null,
+            appPath: null,
+            cloudSessionId: null,
+            artifacts: null,
+            listDevices: false,
+            listDesktopApps,
+            desktopApp,
+            desktopWindow,
+            desktopInput,
+        };
+    }
+    if (args["desktop-app"] != null || args["desktop-window"] != null || args["desktop-input"] != null || args["list-desktop-apps"] != null) {
+        throw new Error("--desktop-app, --desktop-window, --desktop-input, and --list-desktop-apps can only be used with --target desktop.");
+    }
     if (args["session-mode"] != null) {
         throw new Error("--session-mode can only be used with --target web.");
     }

package/build/src/commands/help.js

@@ -15,11 +15,16 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Usage:' });
     addOutput({ type: 'info', text: '  droid-cua --avd <device-name> [options]' });
+    addOutput({ type: 'info', text: '  droid-cua run <file-or-directory...> [options]' });
     addOutput({ type: 'info', text: '  droid-cua --target web --instructions <file> [options]' });
     addOutput({ type: 'info', text: '  droid-cua --device-source lambdatest --instructions <file> [options]' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Options:' });
-    addOutput({ type: 'info', text: '  --target <kind>        Target kind: mobile (default) or web' });
+    addOutput({ type: 'info', text: '  --target <kind>        Target kind: mobile (default), web, or desktop' });
+    addOutput({ type: 'info', text: '  --desktop-app <id>     Running desktop app ID or exact name' });
+    addOutput({ type: 'info', text: '  --desktop-window <id>  Window ID when the app has multiple windows' });
+    addOutput({ type: 'info', text: '  --desktop-input <mode> Physical pointer (default) or cursor-free accessibility' });
+    addOutput({ type: 'info', text: '  --list-desktop-apps    List running desktop apps and windows' });
     addOutput({ type: 'info', text: '  --avd <name>           Device name (Android device ID/serial or iOS Simulator)' });
     addOutput({ type: 'info', text: '  --platform <platform>  Force platform: android or ios' });
     addOutput({ type: 'info', text: '  --device-source <src>  Mobile source: local (default) or lambdatest' });
@@ -28,7 +33,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --app <path>           LambdaTest app package path (.apk or .ipa)' });
     addOutput({ type: 'info', text: '  --artifacts <kinds>    Loadmill Cloud: download video,deviceLog,appiumLog (or all) after the run' });
     addOutput({ type: 'info', text: '  --cloud-session-id <id> Loadmill Cloud: reconnect/retroactive-fetch by mtc_ id (implies --device-source loadmill-cloud)' });
-    addOutput({ type: 'info', text: '  --instructions <file>  Run test file in headless mode' });
+    addOutput({ type: 'info', text: '  --instructions <path>  Legacy headless test file or directory selector' });
     addOutput({ type: 'info', text: '  --scenario <prompt>    Generate and run scenario cases for headless instructions' });
     addOutput({ type: 'info', text: '  --test-data <file>     Optional test-data.md file for --scenario' });
     addOutput({ type: 'info', text: '  --secrets <pairs>      Runtime typing secrets as KEY=value,OTHER=value' });
@@ -88,6 +93,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  droid-cua --avd avd:Pixel_8_API_35       (Launch Android AVD then connect)' });
     addOutput({ type: 'info', text: '  droid-cua --avd "iPhone 16"             (iOS Simulator, auto-detected)' });
     addOutput({ type: 'info', text: '  droid-cua --platform ios --avd MySim    (Force iOS platform)' });
+    addOutput({ type: 'info', text: '  droid-cua run tests/login.dcua admin/refund.dcua --context app/context.md' });
     addOutput({ type: 'info', text: '  droid-cua --instructions tests/login.dcua --context app/context.md' });
     addOutput({ type: 'info', text: '  droid-cua --instructions tests --test-data tests/test-data.md --scenario "Run every checkout test with each buyer account"' });
     addOutput({ type: 'info', text: '  droid-cua --target web --instructions tests/search.dcua' });

package/build/src/core/session.js

@@ -13,7 +13,11 @@ export class Session {
         this.browserName = targetInfo?.browserName || null;
         this.currentUrl = targetInfo?.currentUrl || null;
         this.sessionMode = targetInfo?.sessionMode || null;
-        this.targetKind = targetInfo?.platform === "browser" ? "browser" : "mobile";
+        this.targetKind = targetInfo?.platform === "browser"
+            ? "browser"
+            : targetInfo?.platform === "desktop"
+                ? "desktop"
+                : "mobile";
         this.messages = [];
         this.previousResponseId = null;
         this.transcript = [];

package/build/src/core/test-data-import.js

@@ -1,13 +1,13 @@
 import path from "path";
 import { resolveEffectiveHelperModel } from "./runtime-config.js";
 import { buildTestDataImportMessages } from "../prompts/test-data-import.js";
-export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt"];
+export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt", ".json", ".xml", ".md"];
 export function isSupportedTestDataImportPath(filePath) {
     return TEST_DATA_IMPORT_EXTENSIONS.includes(path.extname(String(filePath ?? "")).toLowerCase());
 }
 export function assertSupportedTestDataImportPath(filePath) {
     if (!isSupportedTestDataImportPath(filePath)) {
-        throw new Error("Test data import supports CSV and TXT files only.");
+        throw new Error("Choose a CSV, TXT, JSON, XML, or Markdown file.");
     }
 }
 function parseTestDataImportContent(content) {

package/build/src/desktop/computer-use.js

@@ -0,0 +1,432 @@
+import { buildActionEventMeta } from "../device/action-event-meta.js";
+import { buildTypeActionPayload, buildTypeActionText, } from "../device/sensitive-action.js";
+import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+const SCREENSHOT_WIDTH = 1440;
+const ACCESSIBILITY_HIT_TOLERANCE = 12;
+const MACOS_CAPTURE_INSETS = { left: 50, right: 50, top: 38, bottom: 62 };
+const SUPPORTED_PLATFORMS = new Set(["darwin", "win32"]);
+const INPUT_MODES = new Set(["accessibility", "physical"]);
+function parseJsonResult(result, label) {
+    if (result?.isError) {
+        throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
+    }
+    if (result?.structuredContent) {
+        return result.structuredContent;
+    }
+    const text = result?.content?.find((item) => item.type === "text")?.text;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`${label} returned an invalid response.`);
+    }
+}
+function assertResult(result, label) {
+    if (result?.isError) {
+        throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
+    }
+}
+function sameBounds(left, right) {
+    return left && right && left.x === right.x && left.y === right.y && left.width === right.width && left.height === right.height;
+}
+function getScreenshotContentBounds(width, height, windowBounds, platform) {
+    const fullScreenshot = { x: 0, y: 0, width, height };
+    if (platform !== "darwin")
+        return fullScreenshot;
+    const { left, right, top, bottom } = MACOS_CAPTURE_INSETS;
+    const scaleX = width / (windowBounds.width + left + right);
+    const scaleY = height / (windowBounds.height + top + bottom);
+    if (Math.abs(scaleX - scaleY) / Math.max(scaleX, scaleY) > 0.02)
+        return fullScreenshot;
+    // ponytail: computer-use-mcp v7 omits capture bounds; remove this calibration when upstream exposes them.
+    return {
+        x: left * scaleX,
+        y: top * scaleY,
+        width: windowBounds.width * scaleX,
+        height: windowBounds.height * scaleY,
+    };
+}
+function formatPermissionError(error, platform) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (platform === "darwin" && /permission|not authorized|screencapture|screen recording|cgwindow|axerror/i.test(message)) {
+        return `${message} Grant Screen & System Audio Recording and Accessibility access to Droid (or your terminal for CLI runs) in System Settings > Privacy & Security, then restart it.`;
+    }
+    if (platform === "win32" && /access denied|elevat|integrity level|could not focus/i.test(message)) {
+        return `${message} Run Droid in an interactive desktop session and at the same integrity level as the target application; elevated Windows applications cannot be controlled by a non-elevated Droid process.`;
+    }
+    return message;
+}
+async function createLibraryClient() {
+    const [{ createComputerUseServer }, { connectInProcess }] = await Promise.all([
+        import("@zavora-ai/computer-use-mcp"),
+        import("@zavora-ai/computer-use-mcp/client"),
+    ]);
+    return connectInProcess(createComputerUseServer({ profile: "ax", vision: true }));
+}
+async function loadAccessibilityDriver() {
+    const { loadNative } = await import("@zavora-ai/computer-use-mcp/host-native");
+    return loadNative();
+}
+function walkAccessibilityTree(root, depth = 0, elements = []) {
+    if (!root || typeof root !== "object")
+        return elements;
+    elements.push({ ...root, depth });
+    for (const child of Array.isArray(root.children) ? root.children : []) {
+        walkAccessibilityTree(child, depth + 1, elements);
+    }
+    return elements;
+}
+function distanceToBounds(bounds, x, y) {
+    if (!bounds || bounds.width <= 0 || bounds.height <= 0)
+        return Infinity;
+    const dx = Math.max(bounds.x - x, 0, x - bounds.x - bounds.width);
+    const dy = Math.max(bounds.y - y, 0, y - bounds.y - bounds.height);
+    return Math.hypot(dx, dy);
+}
+function actionFor(element, actionType, direction = null) {
+    const actions = Array.isArray(element.actions) ? element.actions : [];
+    if (actionType === "click")
+        return actions.find((action) => /press|invoke|click/i.test(action));
+    if (actionType === "double_click")
+        return actions.find((action) => /open|double.?click/i.test(action));
+    if (actionType === "scroll") {
+        const expected = direction === "up" ? /scroll.?up/i
+            : direction === "down" ? /scroll.?down/i
+                : direction === "left" ? /scroll.?left/i
+                    : /scroll.?right/i;
+        return actions.find((action) => expected.test(action));
+    }
+    return null;
+}
+function chooseAccessibilityElement(root, x, y, actionType, direction = null) {
+    const elements = walkAccessibilityTree(root);
+    const candidates = elements
+        .map((element) => ({ element, action: actionFor(element, actionType, direction), distance: distanceToBounds(element.bounds, x, y) }))
+        .filter(({ element, action, distance }) => action && element.role && String(element.label || "").trim() && distance <= ACCESSIBILITY_HIT_TOLERANCE)
+        .sort((left, right) => {
+        const leftArea = left.element.bounds.width * left.element.bounds.height;
+        const rightArea = right.element.bounds.width * right.element.bounds.height;
+        return left.distance - right.distance || leftArea - rightArea || right.element.depth - left.element.depth;
+    });
+    const selected = candidates[0];
+    if (!selected)
+        return null;
+    if (selected.distance > 0 && candidates[1]?.distance === selected.distance)
+        return null;
+    const duplicates = elements.filter((element) => element.role === selected.element.role && element.label === selected.element.label);
+    return duplicates.length === 1 ? selected : null;
+}
+function cursorFreeError(actionType) {
+    return `Cursor-free desktop ${actionType.replaceAll("_", " ")} is unavailable at this location because the application did not expose a unique accessibility action. No mouse input was sent. Re-run with --desktop-input physical to allow OS pointer control.`;
+}
+function cursorFreeKeyboardError(actionType) {
+    return `Cursor-free desktop ${actionType.replaceAll("_", " ")} is unavailable because keyboard input requires application focus. No keyboard input was sent. Re-run with --desktop-input physical to allow focused OS input.`;
+}
+function readApplications(result) {
+    const data = parseJsonResult(result, "Application discovery");
+    return Array.isArray(data) ? data : [];
+}
+async function discoverApplications(client) {
+    let applications = readApplications(await client.listRunningApps());
+    if (applications.length === 0) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        applications = readApplications(await client.listRunningApps());
+    }
+    return applications;
+}
+function readWindows(result) {
+    const data = parseJsonResult(result, "Window discovery");
+    return Array.isArray(data) ? data : Array.isArray(data.windows) ? data.windows : [];
+}
+function chooseApplication(apps, requestedApp) {
+    const needle = String(requestedApp || "").trim().toLowerCase();
+    const matches = apps.filter((app) => app.bundleId?.toLowerCase() === needle || app.displayName?.trim().toLowerCase() === needle);
+    if (matches.length === 1)
+        return matches[0];
+    if (matches.length > 1) {
+        throw new Error(`Desktop application "${requestedApp}" is ambiguous. Use its application ID instead.`);
+    }
+    throw new Error(`Desktop application "${requestedApp}" is not running. Use --list-desktop-apps to see available applications.`);
+}
+function chooseWindow(windows, requestedWindowId) {
+    const visible = windows.filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+    if (requestedWindowId != null) {
+        const selected = visible.find((window) => window.windowId === requestedWindowId);
+        if (!selected)
+            throw new Error(`Desktop window ${requestedWindowId} was not found for the selected application.`);
+        return selected;
+    }
+    const focused = visible.filter((window) => window.isFocused);
+    if (focused.length === 1)
+        return focused[0];
+    if (visible.length === 1)
+        return visible[0];
+    if (visible.length === 0)
+        throw new Error("The selected desktop application has no visible windows.");
+    const choices = visible.map((window) => `${window.windowId}: ${window.title || "Untitled"}`).join("\n");
+    throw new Error(`The selected desktop application has multiple windows. Choose one with --desktop-window <id>:\n${choices}`);
+}
+function normalizeShortcut(keys, platform) {
+    if (!Array.isArray(keys) || keys.length === 0)
+        throw new Error("Keypress action is missing keys.");
+    const aliases = {
+        CTRL: "control",
+        CONTROL: "control",
+        CMD: "command",
+        COMMAND: "command",
+        META: platform === "darwin" ? "command" : "win",
+        OPTION: "alt",
+        ALT: "alt",
+        ESC: "escape",
+        RETURN: "enter",
+    };
+    return keys.map((key) => aliases[String(key).toUpperCase()] || String(key).toLowerCase()).join("+");
+}
+/**
+ * @param {{ app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", client?: any, accessibility?: any }} options
+ */
+export async function createDesktopController({ app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", client: injectedClient = null, accessibility: injectedAccessibility = null, } = {}) {
+    if (!SUPPORTED_PLATFORMS.has(platform)) {
+        throw new Error("Desktop control is currently supported only on macOS and Windows.");
+    }
+    if (!requestedApp)
+        throw new Error("--desktop-app is required with --target desktop.");
+    if (!INPUT_MODES.has(inputMode))
+        throw new Error('Desktop input mode must be "accessibility" or "physical".');
+    const client = injectedClient || await createLibraryClient();
+    try {
+        const apps = (await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid));
+        const application = chooseApplication(apps, requestedApp);
+        const windows = readWindows(await client.listWindows(application.bundleId));
+        let targetWindow = chooseWindow(windows, requestedWindowId == null ? null : Number(requestedWindowId));
+        let capture = null;
+        let deviceInfo = null;
+        let accessibility = injectedAccessibility;
+        async function getWindow() {
+            return parseJsonResult(await client.getWindow(targetWindow.windowId), "Window lookup");
+        }
+        async function screenshot() {
+            let window;
+            let result;
+            try {
+                const currentWindows = readWindows(await client.listWindows(application.bundleId));
+                const focusedWindow = currentWindows.find((candidate) => candidate.isFocused && candidate.isOnScreen);
+                if (focusedWindow)
+                    targetWindow = focusedWindow;
+                window = await getWindow();
+                if (!window.isOnScreen)
+                    throw new Error("The selected desktop window is no longer visible.");
+                result = await client.screenshot({
+                    width: SCREENSHOT_WIDTH,
+                    quality: 0,
+                    target_window_id: targetWindow.windowId,
+                });
+                assertResult(result, "Desktop screenshot");
+            }
+            catch (error) {
+                throw new Error(formatPermissionError(error, platform));
+            }
+            const image = result.content?.find((item) => item.type === "image");
+            const sizeText = result.content?.find((item) => item.type === "text")?.text || "";
+            const size = /^(\d+)x(\d+)$/.exec(sizeText.trim());
+            if (!image?.data || !size)
+                throw new Error("Desktop screenshot returned no image or dimensions.");
+            capture = {
+                bounds: { ...window.bounds },
+                width: Number(size[1]),
+                height: Number(size[2]),
+            };
+            capture.contentBounds = getScreenshotContentBounds(capture.width, capture.height, window.bounds, platform);
+            if (deviceInfo) {
+                deviceInfo.device_width = window.bounds.width;
+                deviceInfo.device_height = window.bounds.height;
+                deviceInfo.scaled_width = capture.width;
+                deviceInfo.scaled_height = capture.height;
+                deviceInfo.scale = capture.width / window.bounds.width;
+                deviceInfo.desktopWindowId = targetWindow.windowId;
+            }
+            return image.data;
+        }
+        function scaledPoint(x, y) {
+            if (!capture)
+                throw new Error("Capture the desktop window before executing an action.");
+            const content = capture.contentBounds;
+            return [
+                Math.round(capture.bounds.x + ((x - content.x) * capture.bounds.width) / content.width),
+                Math.round(capture.bounds.y + ((y - content.y) * capture.bounds.height) / content.height),
+            ];
+        }
+        async function prepareAction(requireFocus) {
+            if (!capture)
+                throw new Error("Capture the desktop window before executing an action.");
+            const before = await getWindow();
+            if (!sameBounds(before.bounds, capture.bounds)) {
+                throw new Error("The desktop window moved or resized after the screenshot. A fresh screenshot is required before acting.");
+            }
+            if (!requireFocus)
+                return;
+            const activation = parseJsonResult(await client.activateWindow(targetWindow.windowId, 2000), "Window activation");
+            const focused = await getWindow();
+            if (!activation.activated || !focused.isFocused) {
+                throw new Error("Droid could not focus the selected desktop window, so no input was sent.");
+            }
+        }
+        const targetOptions = () => ({ targetWindowId: targetWindow.windowId, focusStrategy: "strict" });
+        async function performCursorFreeAction(actionType, x, y, direction = null) {
+            const tree = parseJsonResult(await client.getUiTree(targetWindow.windowId, 20), "Accessibility tree");
+            const selected = chooseAccessibilityElement(tree, x, y, actionType, direction);
+            if (!selected)
+                throw new Error(cursorFreeError(actionType));
+            accessibility ||= await loadAccessibilityDriver();
+            const result = accessibility.performAction(targetWindow.windowId, selected.element.role, selected.element.label, selected.action);
+            if (!result?.performed)
+                throw new Error(`${cursorFreeError(actionType)}${result?.reason ? ` Reason: ${result.reason}.` : ""}`);
+        }
+        async function handleAction(_deviceId, action, _scale = 1, context = null) {
+            const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+            const meta = (payload = {}) => buildActionEventMeta({ context, action, platform: "desktop", payload });
+            try {
+                if (context?.shouldStop?.())
+                    return { success: false, error: "Stopped by user" };
+                if (inputMode === "accessibility" && (action.type === "type" || action.type === "keypress")) {
+                    throw new Error(cursorFreeKeyboardError(action.type));
+                }
+                const requiresFocus = inputMode === "physical";
+                if (action.type !== "wait")
+                    await prepareAction(requiresFocus);
+                switch (action.type) {
+                    case "click":
+                    case "double_click": {
+                        const [x, y] = scaledPoint(action.x, action.y);
+                        addOutput({ type: "action", text: action.type === "click" ? "Click" : `Double-clicking at (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
+                        if (inputMode === "accessibility") {
+                            await performCursorFreeAction(action.type, x, y);
+                        }
+                        else {
+                            const result = action.type === "click"
+                                ? await client.click(x, y, application.bundleId, targetOptions())
+                                : await client.doubleClick(x, y, application.bundleId, targetOptions());
+                            assertResult(result, "Desktop click");
+                        }
+                        break;
+                    }
+                    case "move": {
+                        const [x, y] = scaledPoint(action.x, action.y);
+                        addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
+                        if (inputMode === "accessibility")
+                            throw new Error(cursorFreeError(action.type));
+                        assertResult(await client.moveMouse(x, y, application.bundleId, targetOptions()), "Desktop mouse move");
+                        break;
+                    }
+                    case "drag": {
+                        if (!Array.isArray(action.path) || action.path.length < 2)
+                            throw new Error("Drag action requires a path with at least two points.");
+                        const from = scaledPoint(action.path[0].x, action.path[0].y);
+                        const last = action.path[action.path.length - 1];
+                        const to = scaledPoint(last.x, last.y);
+                        addOutput({ type: "action", text: `Dragging from (${from[0]}, ${from[1]}) to (${to[0]}, ${to[1]})`, ...meta({ pathStart: { x: from[0], y: from[1] }, pathEnd: { x: to[0], y: to[1] }, unit: "px" }) });
+                        if (inputMode === "accessibility")
+                            throw new Error(cursorFreeError(action.type));
+                        assertResult(await client.drag(to, from, application.bundleId, targetOptions()), "Desktop drag");
+                        break;
+                    }
+                    case "scroll": {
+                        const [x, y] = scaledPoint(action.x ?? capture.width / 2, action.y ?? capture.height / 2);
+                        const scrollX = Number(action.scroll_x) || 0;
+                        const scrollY = Number(action.scroll_y) || 0;
+                        const horizontal = Math.abs(scrollX) > Math.abs(scrollY);
+                        const direction = horizontal ? (scrollX > 0 ? "right" : "left") : (scrollY > 0 ? "down" : "up");
+                        const amount = Math.max(1, Math.min(10, Math.round(Math.abs(horizontal ? scrollX : scrollY) / 100)));
+                        addOutput({ type: "action", text: `Scrolling ${direction}`, ...meta({ scrollX, scrollY, x, y, unit: "px" }) });
+                        if (inputMode === "accessibility") {
+                            await performCursorFreeAction(action.type, x, y, direction);
+                        }
+                        else {
+                            assertResult(await client.scroll(x, y, direction, amount, application.bundleId, targetOptions()), "Desktop scroll");
+                        }
+                        break;
+                    }
+                    case "type": {
+                        const text = typeof action.text === "string" ? action.text : "";
+                        addOutput({ type: "action", text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
+                        assertResult(await client.type(text, application.bundleId, targetOptions()), "Desktop typing");
+                        break;
+                    }
+                    case "keypress": {
+                        const shortcut = normalizeShortcut(action.keys, platform);
+                        addOutput({ type: "action", text: `Pressing key: ${shortcut}`, ...meta({ keys: action.keys, shortcut }) });
+                        assertResult(await client.key(shortcut, application.bundleId, targetOptions()), "Desktop keypress");
+                        break;
+                    }
+                    case "wait": {
+                        addOutput({ type: "action", text: "Waiting...", ...meta() });
+                        const timeoutMs = Number.isFinite(action.ms) && action.ms >= 0 ? Math.round(action.ms) : getConfiguredStepDelayMs();
+                        await new Promise((resolve) => setTimeout(resolve, timeoutMs));
+                        break;
+                    }
+                    default:
+                        addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+                }
+                return { success: true };
+            }
+            catch (error) {
+                const message = formatPermissionError(error, platform);
+                addOutput({ type: "error", text: message, eventType: "error", actionType: action?.type, payload: { message, platform: "desktop" } });
+                return { success: false, error: message };
+            }
+        }
+        await screenshot();
+        deviceInfo = {
+            platform: "desktop",
+            source: "local",
+            device_name: `${application.displayName} — ${targetWindow.title || "Untitled"}`,
+            device_width: targetWindow.bounds.width,
+            device_height: targetWindow.bounds.height,
+            scaled_width: capture.width,
+            scaled_height: capture.height,
+            scale: capture.width / targetWindow.bounds.width,
+            desktopAppId: application.bundleId,
+            desktopWindowId: targetWindow.windowId,
+            desktopInputMode: inputMode,
+        };
+        return {
+            application,
+            window: targetWindow,
+            deviceId: `desktop:${targetWindow.windowId}`,
+            deviceInfo,
+            getScreenshotAsBase64: screenshot,
+            handleModelAction: handleAction,
+            close: () => client.close(),
+        };
+    }
+    catch (error) {
+        await client.close().catch(() => { });
+        throw error;
+    }
+}
+/**
+ * @param {{ excludePids?: number[], platform?: string, client?: any }} options
+ */
+export async function listDesktopApplications({ excludePids = [], platform = process.platform, client: injectedClient = null } = {}) {
+    if (!SUPPORTED_PLATFORMS.has(platform)) {
+        throw new Error("Desktop control is currently supported only on macOS and Windows.");
+    }
+    const client = injectedClient || await createLibraryClient();
+    try {
+        const apps = (await discoverApplications(client))
+            .filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid))
+            .sort((left, right) => left.displayName.localeCompare(right.displayName));
+        const entries = [];
+        for (const app of apps) {
+            const windows = readWindows(await client.listWindows(app.bundleId))
+                .filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+            if (windows.length > 0)
+                entries.push({ ...app, windows });
+        }
+        return entries;
+    }
+    finally {
+        if (!injectedClient)
+            await client.close();
+    }
+}

package/build/src/integrations/loadmill/client.js

@@ -3,6 +3,20 @@
  */
 import { extractLoadmillSessionUser } from "./session-user.js";
 const DEFAULT_BASE_URL = "https://app.loadmill.com/api";
+/**
+ * @typedef {{
+ *   id: string,
+ *   email: string | null,
+ *   displayName: string | null,
+ *   teamId: string | null,
+ * }} LoadmillSessionUser
+ */
+/**
+ * @typedef {
+ *   | { valid: true, user: LoadmillSessionUser, status: number, requestId?: string }
+ *   | { valid: false, reason: "missing" | "invalid" | "unavailable" | "malformed", status?: number, requestId?: string }
+ * } ApiTokenValidationResult
+ */
 function normalizeApiBaseUrl(rawBaseUrl) {
     const candidate = (rawBaseUrl || DEFAULT_BASE_URL).trim();
     if (!candidate) {
@@ -150,6 +164,7 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
             body: JSON.stringify(payload),
         });
     }
+    /** @returns {Promise<ApiTokenValidationResult>} */
     async function validateApiToken() {
         const resolvedToken = getApiToken();
         if (!resolvedToken) {
@@ -162,20 +177,25 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
                 Authorization: `Bearer ${resolvedToken}`,
             },
         });
+        const requestId = response.headers.get("x-request-id") || response.headers.get("request-id") || undefined;
+        const responseMetadata = {
+            status: response.status,
+            ...(requestId ? { requestId } : {}),
+        };
         if (response.status === 401 || response.status === 403) {
-            return { valid: false, reason: "invalid", status: response.status };
+            return { valid: false, reason: "invalid", ...responseMetadata };
         }
         if (!response.ok) {
-            return { valid: false, reason: "unavailable", status: response.status };
+            return { valid: false, reason: "unavailable", ...responseMetadata };
         }
         try {
             const user = extractLoadmillSessionUser(await response.json());
             return user
-                ? { valid: true, user }
-                : { valid: false, reason: "unavailable", status: response.status };
+                ? { valid: true, user, ...responseMetadata }
+                : { valid: false, reason: "malformed", ...responseMetadata };
         }
         catch {
-            return { valid: false, reason: "unavailable", status: response.status };
+            return { valid: false, reason: "malformed", ...responseMetadata };
         }
     }
     async function getCurrentUser() {