@loadmill/droid-cua 2.33.0 → 2.35.0

package/build/src/desktop/computer-use.js

@@ -0,0 +1,432 @@
+import { buildActionEventMeta } from "../device/action-event-meta.js";
+import { buildTypeActionPayload, buildTypeActionText, } from "../device/sensitive-action.js";
+import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+const SCREENSHOT_WIDTH = 1440;
+const ACCESSIBILITY_HIT_TOLERANCE = 12;
+const MACOS_CAPTURE_INSETS = { left: 50, right: 50, top: 38, bottom: 62 };
+const SUPPORTED_PLATFORMS = new Set(["darwin", "win32"]);
+const INPUT_MODES = new Set(["accessibility", "physical"]);
+function parseJsonResult(result, label) {
+    if (result?.isError) {
+        throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
+    }
+    if (result?.structuredContent) {
+        return result.structuredContent;
+    }
+    const text = result?.content?.find((item) => item.type === "text")?.text;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`${label} returned an invalid response.`);
+    }
+}
+function assertResult(result, label) {
+    if (result?.isError) {
+        throw new Error(result.content?.find((item) => item.type === "text")?.text || `${label} failed.`);
+    }
+}
+function sameBounds(left, right) {
+    return left && right && left.x === right.x && left.y === right.y && left.width === right.width && left.height === right.height;
+}
+function getScreenshotContentBounds(width, height, windowBounds, platform) {
+    const fullScreenshot = { x: 0, y: 0, width, height };
+    if (platform !== "darwin")
+        return fullScreenshot;
+    const { left, right, top, bottom } = MACOS_CAPTURE_INSETS;
+    const scaleX = width / (windowBounds.width + left + right);
+    const scaleY = height / (windowBounds.height + top + bottom);
+    if (Math.abs(scaleX - scaleY) / Math.max(scaleX, scaleY) > 0.02)
+        return fullScreenshot;
+    // ponytail: computer-use-mcp v7 omits capture bounds; remove this calibration when upstream exposes them.
+    return {
+        x: left * scaleX,
+        y: top * scaleY,
+        width: windowBounds.width * scaleX,
+        height: windowBounds.height * scaleY,
+    };
+}
+function formatPermissionError(error, platform) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (platform === "darwin" && /permission|not authorized|screencapture|screen recording|cgwindow|axerror/i.test(message)) {
+        return `${message} Grant Screen & System Audio Recording and Accessibility access to Droid (or your terminal for CLI runs) in System Settings > Privacy & Security, then restart it.`;
+    }
+    if (platform === "win32" && /access denied|elevat|integrity level|could not focus/i.test(message)) {
+        return `${message} Run Droid in an interactive desktop session and at the same integrity level as the target application; elevated Windows applications cannot be controlled by a non-elevated Droid process.`;
+    }
+    return message;
+}
+async function createLibraryClient() {
+    const [{ createComputerUseServer }, { connectInProcess }] = await Promise.all([
+        import("@zavora-ai/computer-use-mcp"),
+        import("@zavora-ai/computer-use-mcp/client"),
+    ]);
+    return connectInProcess(createComputerUseServer({ profile: "ax", vision: true }));
+}
+async function loadAccessibilityDriver() {
+    const { loadNative } = await import("@zavora-ai/computer-use-mcp/host-native");
+    return loadNative();
+}
+function walkAccessibilityTree(root, depth = 0, elements = []) {
+    if (!root || typeof root !== "object")
+        return elements;
+    elements.push({ ...root, depth });
+    for (const child of Array.isArray(root.children) ? root.children : []) {
+        walkAccessibilityTree(child, depth + 1, elements);
+    }
+    return elements;
+}
+function distanceToBounds(bounds, x, y) {
+    if (!bounds || bounds.width <= 0 || bounds.height <= 0)
+        return Infinity;
+    const dx = Math.max(bounds.x - x, 0, x - bounds.x - bounds.width);
+    const dy = Math.max(bounds.y - y, 0, y - bounds.y - bounds.height);
+    return Math.hypot(dx, dy);
+}
+function actionFor(element, actionType, direction = null) {
+    const actions = Array.isArray(element.actions) ? element.actions : [];
+    if (actionType === "click")
+        return actions.find((action) => /press|invoke|click/i.test(action));
+    if (actionType === "double_click")
+        return actions.find((action) => /open|double.?click/i.test(action));
+    if (actionType === "scroll") {
+        const expected = direction === "up" ? /scroll.?up/i
+            : direction === "down" ? /scroll.?down/i
+                : direction === "left" ? /scroll.?left/i
+                    : /scroll.?right/i;
+        return actions.find((action) => expected.test(action));
+    }
+    return null;
+}
+function chooseAccessibilityElement(root, x, y, actionType, direction = null) {
+    const elements = walkAccessibilityTree(root);
+    const candidates = elements
+        .map((element) => ({ element, action: actionFor(element, actionType, direction), distance: distanceToBounds(element.bounds, x, y) }))
+        .filter(({ element, action, distance }) => action && element.role && String(element.label || "").trim() && distance <= ACCESSIBILITY_HIT_TOLERANCE)
+        .sort((left, right) => {
+        const leftArea = left.element.bounds.width * left.element.bounds.height;
+        const rightArea = right.element.bounds.width * right.element.bounds.height;
+        return left.distance - right.distance || leftArea - rightArea || right.element.depth - left.element.depth;
+    });
+    const selected = candidates[0];
+    if (!selected)
+        return null;
+    if (selected.distance > 0 && candidates[1]?.distance === selected.distance)
+        return null;
+    const duplicates = elements.filter((element) => element.role === selected.element.role && element.label === selected.element.label);
+    return duplicates.length === 1 ? selected : null;
+}
+function cursorFreeError(actionType) {
+    return `Cursor-free desktop ${actionType.replaceAll("_", " ")} is unavailable at this location because the application did not expose a unique accessibility action. No mouse input was sent. Re-run with --desktop-input physical to allow OS pointer control.`;
+}
+function cursorFreeKeyboardError(actionType) {
+    return `Cursor-free desktop ${actionType.replaceAll("_", " ")} is unavailable because keyboard input requires application focus. No keyboard input was sent. Re-run with --desktop-input physical to allow focused OS input.`;
+}
+function readApplications(result) {
+    const data = parseJsonResult(result, "Application discovery");
+    return Array.isArray(data) ? data : [];
+}
+async function discoverApplications(client) {
+    let applications = readApplications(await client.listRunningApps());
+    if (applications.length === 0) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        applications = readApplications(await client.listRunningApps());
+    }
+    return applications;
+}
+function readWindows(result) {
+    const data = parseJsonResult(result, "Window discovery");
+    return Array.isArray(data) ? data : Array.isArray(data.windows) ? data.windows : [];
+}
+function chooseApplication(apps, requestedApp) {
+    const needle = String(requestedApp || "").trim().toLowerCase();
+    const matches = apps.filter((app) => app.bundleId?.toLowerCase() === needle || app.displayName?.trim().toLowerCase() === needle);
+    if (matches.length === 1)
+        return matches[0];
+    if (matches.length > 1) {
+        throw new Error(`Desktop application "${requestedApp}" is ambiguous. Use its application ID instead.`);
+    }
+    throw new Error(`Desktop application "${requestedApp}" is not running. Use --list-desktop-apps to see available applications.`);
+}
+function chooseWindow(windows, requestedWindowId) {
+    const visible = windows.filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+    if (requestedWindowId != null) {
+        const selected = visible.find((window) => window.windowId === requestedWindowId);
+        if (!selected)
+            throw new Error(`Desktop window ${requestedWindowId} was not found for the selected application.`);
+        return selected;
+    }
+    const focused = visible.filter((window) => window.isFocused);
+    if (focused.length === 1)
+        return focused[0];
+    if (visible.length === 1)
+        return visible[0];
+    if (visible.length === 0)
+        throw new Error("The selected desktop application has no visible windows.");
+    const choices = visible.map((window) => `${window.windowId}: ${window.title || "Untitled"}`).join("\n");
+    throw new Error(`The selected desktop application has multiple windows. Choose one with --desktop-window <id>:\n${choices}`);
+}
+function normalizeShortcut(keys, platform) {
+    if (!Array.isArray(keys) || keys.length === 0)
+        throw new Error("Keypress action is missing keys.");
+    const aliases = {
+        CTRL: "control",
+        CONTROL: "control",
+        CMD: "command",
+        COMMAND: "command",
+        META: platform === "darwin" ? "command" : "win",
+        OPTION: "alt",
+        ALT: "alt",
+        ESC: "escape",
+        RETURN: "enter",
+    };
+    return keys.map((key) => aliases[String(key).toUpperCase()] || String(key).toLowerCase()).join("+");
+}
+/**
+ * @param {{ app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", client?: any, accessibility?: any }} options
+ */
+export async function createDesktopController({ app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", client: injectedClient = null, accessibility: injectedAccessibility = null, } = {}) {
+    if (!SUPPORTED_PLATFORMS.has(platform)) {
+        throw new Error("Desktop control is currently supported only on macOS and Windows.");
+    }
+    if (!requestedApp)
+        throw new Error("--desktop-app is required with --target desktop.");
+    if (!INPUT_MODES.has(inputMode))
+        throw new Error('Desktop input mode must be "accessibility" or "physical".');
+    const client = injectedClient || await createLibraryClient();
+    try {
+        const apps = (await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid));
+        const application = chooseApplication(apps, requestedApp);
+        const windows = readWindows(await client.listWindows(application.bundleId));
+        let targetWindow = chooseWindow(windows, requestedWindowId == null ? null : Number(requestedWindowId));
+        let capture = null;
+        let deviceInfo = null;
+        let accessibility = injectedAccessibility;
+        async function getWindow() {
+            return parseJsonResult(await client.getWindow(targetWindow.windowId), "Window lookup");
+        }
+        async function screenshot() {
+            let window;
+            let result;
+            try {
+                const currentWindows = readWindows(await client.listWindows(application.bundleId));
+                const focusedWindow = currentWindows.find((candidate) => candidate.isFocused && candidate.isOnScreen);
+                if (focusedWindow)
+                    targetWindow = focusedWindow;
+                window = await getWindow();
+                if (!window.isOnScreen)
+                    throw new Error("The selected desktop window is no longer visible.");
+                result = await client.screenshot({
+                    width: SCREENSHOT_WIDTH,
+                    quality: 0,
+                    target_window_id: targetWindow.windowId,
+                });
+                assertResult(result, "Desktop screenshot");
+            }
+            catch (error) {
+                throw new Error(formatPermissionError(error, platform));
+            }
+            const image = result.content?.find((item) => item.type === "image");
+            const sizeText = result.content?.find((item) => item.type === "text")?.text || "";
+            const size = /^(\d+)x(\d+)$/.exec(sizeText.trim());
+            if (!image?.data || !size)
+                throw new Error("Desktop screenshot returned no image or dimensions.");
+            capture = {
+                bounds: { ...window.bounds },
+                width: Number(size[1]),
+                height: Number(size[2]),
+            };
+            capture.contentBounds = getScreenshotContentBounds(capture.width, capture.height, window.bounds, platform);
+            if (deviceInfo) {
+                deviceInfo.device_width = window.bounds.width;
+                deviceInfo.device_height = window.bounds.height;
+                deviceInfo.scaled_width = capture.width;
+                deviceInfo.scaled_height = capture.height;
+                deviceInfo.scale = capture.width / window.bounds.width;
+                deviceInfo.desktopWindowId = targetWindow.windowId;
+            }
+            return image.data;
+        }
+        function scaledPoint(x, y) {
+            if (!capture)
+                throw new Error("Capture the desktop window before executing an action.");
+            const content = capture.contentBounds;
+            return [
+                Math.round(capture.bounds.x + ((x - content.x) * capture.bounds.width) / content.width),
+                Math.round(capture.bounds.y + ((y - content.y) * capture.bounds.height) / content.height),
+            ];
+        }
+        async function prepareAction(requireFocus) {
+            if (!capture)
+                throw new Error("Capture the desktop window before executing an action.");
+            const before = await getWindow();
+            if (!sameBounds(before.bounds, capture.bounds)) {
+                throw new Error("The desktop window moved or resized after the screenshot. A fresh screenshot is required before acting.");
+            }
+            if (!requireFocus)
+                return;
+            const activation = parseJsonResult(await client.activateWindow(targetWindow.windowId, 2000), "Window activation");
+            const focused = await getWindow();
+            if (!activation.activated || !focused.isFocused) {
+                throw new Error("Droid could not focus the selected desktop window, so no input was sent.");
+            }
+        }
+        const targetOptions = () => ({ targetWindowId: targetWindow.windowId, focusStrategy: "strict" });
+        async function performCursorFreeAction(actionType, x, y, direction = null) {
+            const tree = parseJsonResult(await client.getUiTree(targetWindow.windowId, 20), "Accessibility tree");
+            const selected = chooseAccessibilityElement(tree, x, y, actionType, direction);
+            if (!selected)
+                throw new Error(cursorFreeError(actionType));
+            accessibility ||= await loadAccessibilityDriver();
+            const result = accessibility.performAction(targetWindow.windowId, selected.element.role, selected.element.label, selected.action);
+            if (!result?.performed)
+                throw new Error(`${cursorFreeError(actionType)}${result?.reason ? ` Reason: ${result.reason}.` : ""}`);
+        }
+        async function handleAction(_deviceId, action, _scale = 1, context = null) {
+            const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+            const meta = (payload = {}) => buildActionEventMeta({ context, action, platform: "desktop", payload });
+            try {
+                if (context?.shouldStop?.())
+                    return { success: false, error: "Stopped by user" };
+                if (inputMode === "accessibility" && (action.type === "type" || action.type === "keypress")) {
+                    throw new Error(cursorFreeKeyboardError(action.type));
+                }
+                const requiresFocus = inputMode === "physical";
+                if (action.type !== "wait")
+                    await prepareAction(requiresFocus);
+                switch (action.type) {
+                    case "click":
+                    case "double_click": {
+                        const [x, y] = scaledPoint(action.x, action.y);
+                        addOutput({ type: "action", text: action.type === "click" ? "Click" : `Double-clicking at (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
+                        if (inputMode === "accessibility") {
+                            await performCursorFreeAction(action.type, x, y);
+                        }
+                        else {
+                            const result = action.type === "click"
+                                ? await client.click(x, y, application.bundleId, targetOptions())
+                                : await client.doubleClick(x, y, application.bundleId, targetOptions());
+                            assertResult(result, "Desktop click");
+                        }
+                        break;
+                    }
+                    case "move": {
+                        const [x, y] = scaledPoint(action.x, action.y);
+                        addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
+                        if (inputMode === "accessibility")
+                            throw new Error(cursorFreeError(action.type));
+                        assertResult(await client.moveMouse(x, y, application.bundleId, targetOptions()), "Desktop mouse move");
+                        break;
+                    }
+                    case "drag": {
+                        if (!Array.isArray(action.path) || action.path.length < 2)
+                            throw new Error("Drag action requires a path with at least two points.");
+                        const from = scaledPoint(action.path[0].x, action.path[0].y);
+                        const last = action.path[action.path.length - 1];
+                        const to = scaledPoint(last.x, last.y);
+                        addOutput({ type: "action", text: `Dragging from (${from[0]}, ${from[1]}) to (${to[0]}, ${to[1]})`, ...meta({ pathStart: { x: from[0], y: from[1] }, pathEnd: { x: to[0], y: to[1] }, unit: "px" }) });
+                        if (inputMode === "accessibility")
+                            throw new Error(cursorFreeError(action.type));
+                        assertResult(await client.drag(to, from, application.bundleId, targetOptions()), "Desktop drag");
+                        break;
+                    }
+                    case "scroll": {
+                        const [x, y] = scaledPoint(action.x ?? capture.width / 2, action.y ?? capture.height / 2);
+                        const scrollX = Number(action.scroll_x) || 0;
+                        const scrollY = Number(action.scroll_y) || 0;
+                        const horizontal = Math.abs(scrollX) > Math.abs(scrollY);
+                        const direction = horizontal ? (scrollX > 0 ? "right" : "left") : (scrollY > 0 ? "down" : "up");
+                        const amount = Math.max(1, Math.min(10, Math.round(Math.abs(horizontal ? scrollX : scrollY) / 100)));
+                        addOutput({ type: "action", text: `Scrolling ${direction}`, ...meta({ scrollX, scrollY, x, y, unit: "px" }) });
+                        if (inputMode === "accessibility") {
+                            await performCursorFreeAction(action.type, x, y, direction);
+                        }
+                        else {
+                            assertResult(await client.scroll(x, y, direction, amount, application.bundleId, targetOptions()), "Desktop scroll");
+                        }
+                        break;
+                    }
+                    case "type": {
+                        const text = typeof action.text === "string" ? action.text : "";
+                        addOutput({ type: "action", text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
+                        assertResult(await client.type(text, application.bundleId, targetOptions()), "Desktop typing");
+                        break;
+                    }
+                    case "keypress": {
+                        const shortcut = normalizeShortcut(action.keys, platform);
+                        addOutput({ type: "action", text: `Pressing key: ${shortcut}`, ...meta({ keys: action.keys, shortcut }) });
+                        assertResult(await client.key(shortcut, application.bundleId, targetOptions()), "Desktop keypress");
+                        break;
+                    }
+                    case "wait": {
+                        addOutput({ type: "action", text: "Waiting...", ...meta() });
+                        const timeoutMs = Number.isFinite(action.ms) && action.ms >= 0 ? Math.round(action.ms) : getConfiguredStepDelayMs();
+                        await new Promise((resolve) => setTimeout(resolve, timeoutMs));
+                        break;
+                    }
+                    default:
+                        addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+                }
+                return { success: true };
+            }
+            catch (error) {
+                const message = formatPermissionError(error, platform);
+                addOutput({ type: "error", text: message, eventType: "error", actionType: action?.type, payload: { message, platform: "desktop" } });
+                return { success: false, error: message };
+            }
+        }
+        await screenshot();
+        deviceInfo = {
+            platform: "desktop",
+            source: "local",
+            device_name: `${application.displayName} — ${targetWindow.title || "Untitled"}`,
+            device_width: targetWindow.bounds.width,
+            device_height: targetWindow.bounds.height,
+            scaled_width: capture.width,
+            scaled_height: capture.height,
+            scale: capture.width / targetWindow.bounds.width,
+            desktopAppId: application.bundleId,
+            desktopWindowId: targetWindow.windowId,
+            desktopInputMode: inputMode,
+        };
+        return {
+            application,
+            window: targetWindow,
+            deviceId: `desktop:${targetWindow.windowId}`,
+            deviceInfo,
+            getScreenshotAsBase64: screenshot,
+            handleModelAction: handleAction,
+            close: () => client.close(),
+        };
+    }
+    catch (error) {
+        await client.close().catch(() => { });
+        throw error;
+    }
+}
+/**
+ * @param {{ excludePids?: number[], platform?: string, client?: any }} options
+ */
+export async function listDesktopApplications({ excludePids = [], platform = process.platform, client: injectedClient = null } = {}) {
+    if (!SUPPORTED_PLATFORMS.has(platform)) {
+        throw new Error("Desktop control is currently supported only on macOS and Windows.");
+    }
+    const client = injectedClient || await createLibraryClient();
+    try {
+        const apps = (await discoverApplications(client))
+            .filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid))
+            .sort((left, right) => left.displayName.localeCompare(right.displayName));
+        const entries = [];
+        for (const app of apps) {
+            const windows = readWindows(await client.listWindows(app.bundleId))
+                .filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+            if (windows.length > 0)
+                entries.push({ ...app, windows });
+        }
+        return entries;
+    }
+    finally {
+        if (!injectedClient)
+            await client.close();
+    }
+}

package/build/src/device/cloud/actions.js

@@ -77,12 +77,22 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 break;
             }
             case "type": {
+                const typingSlowly = platform === "ios" && action.typingMode === "slow";
+                const actionText = buildTypeActionText(action, action.text);
                 addOutput({
                     type: "action",
-                    text: buildTypeActionText(action, action.text),
-                    ...meta(buildTypeActionPayload(action, action.text))
+                    text: typingSlowly ? actionText.replace(":", " slowly:") : actionText,
+                    ...meta({
+                        ...buildTypeActionPayload(action, action.text),
+                        ...(typingSlowly ? { typingMode: "slow" } : {})
+                    })
                 });
-                await session.client.type(session.sessionId, action.text);
+                if (typingSlowly) {
+                    await session.client.typeSlowly(session.sessionId, action.text);
+                }
+                else {
+                    await session.client.type(session.sessionId, action.text);
+                }
                 break;
             }
             case "scroll": {

package/build/src/device/cloud/cloud-appium-client.js

@@ -119,6 +119,12 @@ export class CloudAppiumClient {
             ]
         });
     }
+    async typeSlowly(sessionId, text) {
+        await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: keys",
+            args: [{ keys: [...text] }]
+        });
+    }
     async isKeyboardShown(sessionId) {
         const response = await this.request("POST", `/session/${sessionId}/execute/sync`, {
             script: "mobile: isKeyboardShown",

package/build/src/device/ios/actions.js

@@ -12,17 +12,19 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
-import { closeAppiumKeyboard } from "../keyboard.js";
-export async function closeKeyboard() {
+import { closeAppiumKeyboardWithTap } from "../keyboard.js";
+export async function closeKeyboard(_deviceId, { dismissalPoint = null, scale = 1 } = {}) {
     const session = getActiveSession();
     if (!session) {
         throw new Error("No active iOS session");
     }
-    // The local simulator supports Appium's keyboard command reliably, so it
-    // needs neither a model-selected tap nor a direct Return keypress.
-    return closeAppiumKeyboard({
+    const dpr = getDevicePixelRatio();
+    const tap = dismissalPoint
+        ? () => appium.tap(session.sessionId, Math.round((dismissalPoint.x / scale) / dpr), Math.round((dismissalPoint.y / scale) / dpr))
+        : null;
+    return closeAppiumKeyboardWithTap({
         isKeyboardShown: () => appium.isKeyboardShown(session.sessionId),
-        hideKeyboard: () => appium.hideKeyboard(session.sessionId),
+        tap,
     });
 }
 /**
@@ -74,12 +76,22 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 break;
             }
             case "type": {
+                const typingSlowly = action.typingMode === "slow";
+                const actionText = buildTypeActionText(action, action.text);
                 addOutput({
                     type: "action",
-                    text: buildTypeActionText(action, action.text),
-                    ...meta(buildTypeActionPayload(action, action.text))
+                    text: typingSlowly ? actionText.replace(":", " slowly:") : actionText,
+                    ...meta({
+                        ...buildTypeActionPayload(action, action.text),
+                        ...(typingSlowly ? { typingMode: "slow" } : {})
+                    })
                 });
-                await appium.type(session.sessionId, action.text);
+                if (typingSlowly) {
+                    await appium.typeSlowly(session.sessionId, action.text);
+                }
+                else {
+                    await appium.type(session.sessionId, action.text);
+                }
                 break;
             }
             case "scroll": {

package/build/src/device/ios/appium-client.js

@@ -123,6 +123,18 @@ export async function type(sessionId, text) {
         ],
     });
 }
+/**
+ * Type text one key at a time through XCUITest's keyboard input API.
+ * @param {string} sessionId
+ * @param {string} text
+ * @returns {Promise<void>}
+ */
+export async function typeSlowly(sessionId, text) {
+    await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: keys",
+        args: [{ keys: [...text] }],
+    });
+}
 export async function isKeyboardShown(sessionId) {
     const response = await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
         script: "mobile: isKeyboardShown",
@@ -131,9 +143,8 @@ export async function isKeyboardShown(sessionId) {
     return response.value === true;
 }
 export async function hideKeyboard(sessionId) {
-    // This XCUITest form is used only by the locally validated simulator path.
-    // Real cloud iOS devices use a guarded tap instead because this command was
-    // unreliable there and sending Return directly can submit the current form.
+    // The CUA close_keyboard path uses a model-selected tap because this command
+    // may send Return. Keep the low-level command available for direct callers.
     await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
         script: "mobile: hideKeyboard",
         args: [{ keys: ["return"] }],

package/build/src/device/keyboard.js

@@ -6,10 +6,9 @@ import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
  *   navigate away. Android's mInputShown flag stays true when a hardware
  *   keyboard suppresses the software keyboard, so that configuration is
  *   treated as already closed before Back is considered.
- * - Local iOS and cloud Android use Appium's native keyboard commands, which
- *   worked reliably in device testing and do not require choosing a tap target.
- * - Cloud iOS uses a model-selected tap. XCUITest's hideKeyboard command was
- *   unreliable on the tested real devices, while Return/newline could submit a
+ * - Cloud Android uses Appium's native keyboard command.
+ * - Local and cloud iOS use a model-selected tap. XCUITest's hideKeyboard
+ *   command was unreliable in testing, while Return/newline could submit a
  *   form. The caller therefore supplies a trusted dismissal point instead.
  *
  * Every strategy follows the same query -> dismiss -> wait -> verify shape and