@loadmill/droid-cua 1.1.1 → 2.0.0

package/build/src/device/android/actions.js

@@ -1,25 +1,81 @@
-import { exec } from "child_process";
-import { promisify } from "util";
 import { logger } from "../../utils/logger.js";
-const execAsync = promisify(exec);
+import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
+import { execAdb } from "./tools.js";
 function adbShell(deviceId, command) {
-    return execAsync(`adb -s ${deviceId} shell "${command}"`);
+    return execAdb(["-s", deviceId, "shell", command]);
+}
+function normalizeMobileKeypress(keys = []) {
+    if (!Array.isArray(keys) || keys.length === 0) {
+        throw new Error("Keypress action is missing keys");
+    }
+    if (keys.length > 1) {
+        throw new Error(`Unsupported mobile key chord: ${keys.join(", ")}. Use taps and text entry instead.`);
+    }
+    const key = String(keys[0]).trim().toUpperCase();
+    const mobileKeyMap = {
+        ESC: "KEYCODE_HOME",
+        ESCAPE: "KEYCODE_HOME",
+        HOME: "KEYCODE_HOME",
+        BACK: "KEYCODE_BACK",
+        ENTER: "KEYCODE_ENTER",
+        RETURN: "KEYCODE_ENTER",
+        BACKSPACE: "KEYCODE_DEL",
+        DELETE: "KEYCODE_DEL",
+        SPACE: "KEYCODE_SPACE"
+    };
+    const mappedKey = mobileKeyMap[key];
+    if (!mappedKey) {
+        throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
+    }
+    return { originalKey: keys[0], mappedKey };
 }
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const meta = (payload = {}) => ({
+        eventType: 'tool_call',
+        actionType: action?.type,
+        runId: context?.runId,
+        stepId: context?.stepId,
+        instructionIndex: context?.instructionIndex,
+        payload: {
+            platform: 'android',
+            ...payload
+        }
+    });
     try {
         const { x, y, x1, y1, x2, y2, text, keys, path } = action;
+        emitDesktopDebug("device.action.execute", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex
+        }, {
+            platform: "android",
+            deviceId,
+            actionType: action?.type,
+            scale,
+            x,
+            y,
+            x1,
+            y1,
+            x2,
+            y2,
+            text: typeof text === "string" ? truncateForDebug(text, 300) : undefined,
+            keyCount: Array.isArray(keys) ? keys.length : 0,
+            pathPoints: Array.isArray(path) ? path.length : 0
+        });
         switch (action.type) {
             case "click":
+            case "double_click":
                 const realX = Math.round(x / scale);
                 const realY = Math.round(y / scale);
-                addOutput({ type: 'action', text: `Clicking at (${realX}, ${realY})` });
+                addOutput({ type: 'action', text: `Clicking at (${realX}, ${realY})`, ...meta({ x: realX, y: realY, unit: 'px' }) });
                 await adbShell(deviceId, `input tap ${realX} ${realY}`);
                 break;
             case "scroll":
                 const scrollX = Math.round(action.scroll_x / scale);
                 const scrollY = Math.round(action.scroll_y / scale);
-                addOutput({ type: 'action', text: `Scrolling by (${scrollX}, ${scrollY})` });
+                addOutput({ type: 'action', text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY, unit: 'px' }) });
                 const startX = 500;
                 const startY = 500;
                 const endX = startX + scrollX;
@@ -34,7 +90,11 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                     const realStartY = Math.round(start.y / scale);
                     const realEndX = Math.round(end.x / scale);
                     const realEndY = Math.round(end.y / scale);
-                    addOutput({ type: 'action', text: `Dragging from (${realStartX}, ${realStartY}) to (${realEndX}, ${realEndY})` });
+                    addOutput({
+                        type: 'action',
+                        text: `Dragging from (${realStartX}, ${realStartY}) to (${realEndX}, ${realEndY})`,
+                        ...meta({ pathStart: { x: realStartX, y: realStartY }, pathEnd: { x: realEndX, y: realEndY }, unit: 'px' })
+                    });
                     await adbShell(deviceId, `input swipe ${realStartX} ${realStartY} ${realEndX} ${realEndY} 500`);
                 }
                 else {
@@ -42,25 +102,19 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 }
                 break;
             case "type":
-                addOutput({ type: 'action', text: `Typing text: ${text}` });
+                addOutput({ type: 'action', text: `Typing text: ${text}`, ...meta({ text }) });
                 const escapedText = text.replace(/(["\\$`])/g, "\\$1").replace(/ /g, "%s");
                 await adbShell(deviceId, `input text "${escapedText}"`);
                 break;
             case "keypress":
-                // Map ESC to Android Home button (since ESC doesn't exist on mobile)
-                const mappedKeys = keys.map(key => {
-                    if (key.toUpperCase() === 'ESC' || key.toUpperCase() === 'ESCAPE') {
-                        return 'KEYCODE_HOME';
-                    }
-                    return key;
-                });
-                addOutput({ type: 'action', text: `Pressing key: ${mappedKeys.join(', ')}` });
-                for (const key of mappedKeys) {
-                    await adbShell(deviceId, `input keyevent ${key}`);
+                {
+                    const { originalKey, mappedKey } = normalizeMobileKeypress(keys);
+                    addOutput({ type: 'action', text: `Pressing key: ${mappedKey}`, ...meta({ keys: [originalKey], mappedKeys: [mappedKey] }) });
+                    await adbShell(deviceId, `input keyevent ${mappedKey}`);
                 }
                 break;
             case "wait":
-                addOutput({ type: 'action', text: 'Waiting...' });
+                addOutput({ type: 'action', text: 'Waiting...', ...meta({}) });
                 await new Promise(res => setTimeout(res, 1000));
                 break;
             default:
@@ -74,8 +128,31 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             message: error.message,
             stack: error.stack
         });
+        emitDesktopDebug("device.error", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex
+        }, {
+            platform: "android",
+            operation: "action.execute",
+            actionType: action?.type,
+            message: error.message
+        });
         // Show user-friendly error message
-        addOutput({ type: 'error', text: `Error executing action: ${error.message}` });
+        addOutput({
+            type: 'error',
+            text: `Error executing action: ${error.message}`,
+            eventType: 'error',
+            actionType: action?.type,
+            runId: context?.runId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex,
+            payload: {
+                message: error.message,
+                platform: 'android'
+            }
+        });
         addOutput({ type: 'info', text: 'Full error details have been logged to the debug log.' });
     }
 }

package/build/src/device/android/connection.js

@@ -1,109 +1,191 @@
-import { exec, spawn } from "child_process";
 import { once } from "events";
-import { promisify } from "util";
 import sharp from "sharp";
 import { logger } from "../../utils/logger.js";
-const execAsync = promisify(exec);
-function wait(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
+import { emitDesktopDebug } from "../../utils/desktop-debug.js";
+import { execAdb, execEmulator, spawnAdb, spawnEmulator } from "./tools.js";
 async function listConnectedDevices() {
-    const { stdout } = await execAsync("adb devices");
+    const { stdout } = await execAdb(["devices"]);
     return stdout
-        .trim()
         .split("\n")
         .slice(1)
-        .map(line => line.split("\t")[0])
-        .filter(id => id.length > 0);
+        .map(line => line.trim())
+        .filter(Boolean)
+        .map(line => {
+        const [id, state] = line.split("\t");
+        return {
+            id: id?.trim() || "",
+            state: state?.trim() || "",
+        };
+    })
+        .filter(entry => entry.id.length > 0);
 }
-async function waitForDeviceConnection(avdName, timeoutMs = 120000) {
-    const deadline = Date.now() + timeoutMs;
-    while (Date.now() < deadline) {
-        const devices = await listConnectedDevices();
-        const match = devices.find(id => id.includes(avdName));
-        if (match)
-            return match;
-        await wait(2000);
+function wait(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function parseTargetToken(rawTarget) {
+    if (!rawTarget)
+        return { kind: "auto" };
+    if (rawTarget.startsWith("adb:")) {
+        return { kind: "adb", serial: rawTarget.slice(4) };
+    }
+    if (rawTarget.startsWith("avd:")) {
+        return { kind: "avd", avdName: rawTarget.slice(4) };
+    }
+    return { kind: "legacy", value: rawTarget };
+}
+async function getEmulatorAvdName(serial) {
+    const candidates = [
+        ["-s", serial, "emu", "avd", "name"],
+        ["-s", serial, "shell", "getprop", "ro.boot.qemu.avd_name"],
+        ["-s", serial, "shell", "getprop", "ro.kernel.qemu.avd_name"],
+        ["-s", serial, "shell", "getprop", "persist.sys.avd_name"],
+    ];
+    for (const args of candidates) {
+        try {
+            const { stdout } = await execAdb(args);
+            const value = stdout.trim();
+            if (value)
+                return value;
+        }
+        catch { }
     }
     return null;
 }
-async function waitForDeviceBoot(deviceId, timeoutMs = 60000) {
+async function waitForWindowServiceReady(deviceId, timeoutMs = 60000) {
     const deadline = Date.now() + timeoutMs;
     while (Date.now() < deadline) {
         try {
-            const { stdout } = await execAsync(`adb -s ${deviceId} shell getprop sys.boot_completed`);
-            if (stdout.trim() === "1")
+            const { stdout } = await execAdb(["-s", deviceId, "shell", "wm", "size"]);
+            if (/Physical size:\s*\d+x\d+/.test(stdout)) {
                 return true;
+            }
         }
         catch { }
         await wait(2000);
     }
     return false;
 }
-/**
- * Get list of available AVDs
- */
-async function listAvailableAVDs() {
+async function findRunningEmulatorByAvdName(avdName) {
+    const devices = await listConnectedDevices();
+    const readyEmulators = devices.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-"));
+    for (const entry of readyEmulators) {
+        const runningAvdName = await getEmulatorAvdName(entry.id);
+        if (runningAvdName === avdName) {
+            return entry.id;
+        }
+    }
+    return null;
+}
+async function launchAndWaitForAvd(avdName, timeoutMs = 60000) {
+    let avdExists = false;
     try {
-        const { stdout } = await execAsync("emulator -list-avds");
-        return stdout.trim().split("\n").filter(name => name.length > 0);
+        const { stdout } = await execEmulator(["-list-avds"]);
+        avdExists = stdout
+            .split("\n")
+            .map((line) => line.trim())
+            .filter(Boolean)
+            .includes(avdName);
+    }
+    catch { }
+    if (!avdExists) {
+        throw new Error(`AVD "${avdName}" was not found. Create it in Android Studio and retry.`);
     }
-    catch {
-        return [];
+    const baseline = await listConnectedDevices();
+    const baselineReadyEmulators = new Set(baseline.filter((entry) => entry.state === "device" && entry.id.startsWith("emulator-")).map((entry) => entry.id));
+    console.log(`Launching Android emulator "${avdName}"...`);
+    const emulatorProcess = spawnEmulator(["-avd", avdName], { detached: true, stdio: "ignore" });
+    emulatorProcess.unref();
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const exactMatch = await findRunningEmulatorByAvdName(avdName);
+        if (exactMatch) {
+            return exactMatch;
+        }
+        const devices = await listConnectedDevices();
+        const newReadyEmulator = devices.find((entry) => entry.state === "device" && entry.id.startsWith("emulator-") && !baselineReadyEmulators.has(entry.id));
+        if (newReadyEmulator) {
+            return newReadyEmulator.id;
+        }
+        await wait(2000);
     }
+    throw new Error("Emulator launch timed out after 60s. Check AVD health and retry.");
 }
-export async function connectToDevice(avdName) {
+export async function connectToDevice(deviceTargetId) {
+    emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "start", targetId: deviceTargetId || null });
+    const parsedTarget = parseTargetToken(deviceTargetId || "");
     const devices = await listConnectedDevices();
-    // If no AVD specified, try to use an already-running emulator or pick the first available
-    if (!avdName) {
-        // Check for already-running emulator
-        for (const id of devices) {
-            if (id.startsWith("emulator-")) {
-                console.log(`Using already-running emulator: ${id}`);
-                return id;
+    const deviceMap = new Map(devices.map((entry) => [entry.id, entry.state]));
+    const readyDevices = devices.filter((entry) => entry.state === "device");
+    if (parsedTarget.kind === "auto") {
+        if (readyDevices.length === 0) {
+            emitDesktopDebug("device.error", "device", {}, {
+                platform: "android",
+                operation: "connect",
+                message: "No ready Android devices detected."
+            });
+            throw new Error("No ready Android devices found. Connect a device (or start an emulator), authorize adb, and retry.");
+        }
+        const selected = readyDevices[0].id;
+        console.log(`Using connected Android device: ${selected}`);
+        emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: selected, reused: true });
+        return selected;
+    }
+    if (parsedTarget.kind === "avd") {
+        const alreadyRunning = await findRunningEmulatorByAvdName(parsedTarget.avdName);
+        if (alreadyRunning) {
+            const ready = await waitForWindowServiceReady(alreadyRunning, 60000);
+            if (!ready) {
+                throw new Error("Emulator launch timed out after 60s. Check AVD health and retry.");
             }
+            console.log(`Using already-running emulator for AVD "${parsedTarget.avdName}": ${alreadyRunning}`);
+            emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: alreadyRunning, reused: true });
+            return alreadyRunning;
         }
-        // No running emulator, pick first available AVD
-        const avds = await listAvailableAVDs();
-        if (avds.length === 0) {
-            console.error("No Android AVDs found. Create one with Android Studio or run:");
-            console.error("  avdmanager create avd -n Pixel_8 -k 'system-images;android-35;google_apis;arm64-v8a'");
-            process.exit(1);
+        const launchedId = await launchAndWaitForAvd(parsedTarget.avdName, 60000);
+        const ready = await waitForWindowServiceReady(launchedId, 60000);
+        if (!ready) {
+            throw new Error("Emulator launch timed out after 60s. Check AVD health and retry.");
         }
-        avdName = avds[0];
-        console.log(`No AVD specified, using first available: ${avdName}`);
-    }
-    for (const id of devices) {
-        if (id.startsWith("emulator-")) {
-            try {
-                const { stdout } = await execAsync(`adb -s ${id} emu avd name`);
-                if (stdout.trim() === avdName) {
-                    console.log(`Emulator ${avdName} is already running as ${id}`);
-                    return id;
-                }
-            }
-            catch { }
+        console.log(`Connected to launched emulator "${parsedTarget.avdName}" as ${launchedId}`);
+        emitDesktopDebug("device.connect", "device", {}, {
+            platform: "android",
+            stage: "success",
+            deviceId: launchedId,
+            reused: false,
+            avdName: parsedTarget.avdName
+        });
+        return launchedId;
+    }
+    const targetId = parsedTarget.kind === "adb" ? parsedTarget.serial : parsedTarget.value;
+    const state = deviceMap.get(targetId);
+    if (state === "device") {
+        const ready = await waitForWindowServiceReady(targetId, 60000);
+        if (!ready) {
+            throw new Error("Selected device is connected but not ready yet. Wait a moment and retry.");
         }
+        console.log(`Using selected Android device: ${targetId}`);
+        emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: targetId, reused: true });
+        return targetId;
     }
-    console.log(`No emulator with AVD "${avdName}" is running. Launching...`);
-    const emulatorProcess = spawn("emulator", ["-avd", avdName], { detached: true, stdio: "ignore" });
-    emulatorProcess.unref();
-    const deviceId = await waitForDeviceConnection("emulator-", 120000);
-    if (!deviceId) {
-        console.error(`Emulator ${avdName} did not appear in time.`);
-        process.exit(1);
+    if (parsedTarget.kind === "legacy" && targetId) {
+        const runningLegacyAvd = await findRunningEmulatorByAvdName(targetId);
+        if (runningLegacyAvd) {
+            emitDesktopDebug("device.connect", "device", {}, { platform: "android", stage: "success", deviceId: runningLegacyAvd, reused: true });
+            return runningLegacyAvd;
+        }
     }
-    console.log(`Device ${deviceId} detected. Waiting for boot...`);
-    const booted = await waitForDeviceBoot(deviceId);
-    if (!booted) {
-        console.error(`Emulator ${avdName} did not finish booting.`);
-        process.exit(1);
+    let message = "Selected device is no longer connected. Refresh and select again.";
+    if (state === "offline") {
+        message = "Device is offline. Reconnect device or restart adb.";
     }
-    console.log(`Emulator ${avdName} is fully booted.`);
-    return deviceId;
+    else if (state === "unauthorized") {
+        message = "Authorize this computer on the device and retry.";
+    }
+    emitDesktopDebug("device.error", "device", {}, { platform: "android", operation: "connect", message, targetId: targetId || deviceTargetId, state: state || "missing" });
+    throw new Error(message);
 }
 export async function getDeviceInfo(deviceId) {
-    const { stdout } = await execAsync(`adb -s ${deviceId} shell wm size`);
+    const { stdout } = await execAdb(["-s", deviceId, "shell", "wm", "size"]);
     const match = stdout.match(/Physical size:\s*(\d+)x(\d+)/);
     if (!match) {
         console.error("Could not get device screen size.");
@@ -114,7 +196,18 @@ export async function getDeviceInfo(deviceId) {
     const scale = width > targetWidth ? targetWidth / width : 1.0;
     const scaledWidth = Math.round(width * scale);
     const scaledHeight = Math.round(height * scale);
+    const [{ stdout: manufacturerRaw }, { stdout: modelRaw }, { stdout: deviceNameRaw }] = await Promise.all([
+        execAdb(["-s", deviceId, "shell", "getprop", "ro.product.manufacturer"]),
+        execAdb(["-s", deviceId, "shell", "getprop", "ro.product.model"]),
+        execAdb(["-s", deviceId, "shell", "getprop", "ro.product.device"])
+    ]);
+    const manufacturer = manufacturerRaw.trim();
+    const model = modelRaw.trim();
+    const deviceName = [manufacturer, model].filter(Boolean).join(" ").trim() || deviceNameRaw.trim() || deviceId;
     return {
+        platform: "android",
+        device_name: deviceName,
+        model: model || deviceNameRaw.trim() || deviceId,
         device_width: width,
         device_height: height,
         scaled_width: scaledWidth,
@@ -123,7 +216,7 @@ export async function getDeviceInfo(deviceId) {
     };
 }
 export async function getScreenshotAsBase64(deviceId, deviceInfo) {
-    const adb = spawn("adb", ["-s", deviceId, "exec-out", "screencap", "-p"]);
+    const adb = spawnAdb(["-s", deviceId, "exec-out", "screencap", "-p"]);
     const chunks = [];
     const stderrChunks = [];
     adb.stdout.on("data", chunk => chunks.push(chunk));
@@ -135,12 +228,14 @@ export async function getScreenshotAsBase64(deviceId, deviceInfo) {
     if (code !== 0) {
         const stderrOutput = Buffer.concat(stderrChunks).toString();
         logger.error(`ADB screencap failed with code ${code}`, { stderr: stderrOutput });
+        emitDesktopDebug("device.error", "device", {}, { platform: "android", operation: "screenshot", deviceId, message: `adb screencap exited with code ${code}` });
         throw new Error(`adb screencap exited with code ${code}`);
     }
     let buffer = Buffer.concat(chunks);
     logger.debug(`Screenshot captured: ${buffer.length} bytes before scaling`);
     if (buffer.length === 0) {
         logger.error('Screenshot buffer is empty!', { deviceId, chunks: chunks.length });
+        emitDesktopDebug("device.error", "device", {}, { platform: "android", operation: "screenshot", deviceId, message: "Screenshot capture returned empty buffer" });
         throw new Error('Screenshot capture returned empty buffer');
     }
     if (deviceInfo.scale < 1.0) {
@@ -150,5 +245,13 @@ export async function getScreenshotAsBase64(deviceId, deviceInfo) {
             .toBuffer();
         logger.debug(`Screenshot scaled: ${buffer.length} bytes after scaling`);
     }
-    return buffer.toString("base64");
+    const base64 = buffer.toString("base64");
+    emitDesktopDebug("device.screenshot", "device", {}, {
+        platform: "android",
+        deviceId,
+        width: deviceInfo?.scaled_width,
+        height: deviceInfo?.scaled_height,
+        base64Length: base64.length
+    });
+    return base64;
 }

package/build/src/device/android/tools.js

@@ -0,0 +1,21 @@
+import { execFile, spawn } from "child_process";
+import { promisify } from "util";
+const execFileAsync = promisify(execFile);
+export function getAdbCommand() {
+    return process.env.DROID_CUA_ANDROID_ADB_PATH?.trim() || "adb";
+}
+export function getEmulatorCommand() {
+    return process.env.DROID_CUA_ANDROID_EMULATOR_PATH?.trim() || "emulator";
+}
+export function execAdb(args, options = {}) {
+    return execFileAsync(getAdbCommand(), args, options);
+}
+export function execEmulator(args, options = {}) {
+    return execFileAsync(getEmulatorCommand(), args, options);
+}
+export function spawnAdb(args, options = {}) {
+    return spawn(getAdbCommand(), args, options);
+}
+export function spawnEmulator(args, options = {}) {
+    return spawn(getEmulatorCommand(), args, options);
+}

package/build/src/device/assertions.js

@@ -54,12 +54,23 @@ export function extractFailureDetails(transcript) {
     const parts = recentTranscript.split("ASSERTION RESULT: FAIL");
     return parts[1]?.trim() || "Could not confidently validate the assertion.";
 }
-export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context) {
+export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context, stepContext = null) {
     const details = extractFailureDetails(transcript);
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
-    addOutput({ type: 'error', text: '❌ ASSERTION FAILED' });
-    addOutput({ type: 'error', text: `Assertion: ${assertionPrompt}` });
-    addOutput({ type: 'error', text: `Details: ${details}` });
+    const meta = {
+        eventType: 'assertion_result',
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        payload: {
+            assertion: assertionPrompt,
+            passed: false,
+            details
+        }
+    };
+    addOutput({ type: 'error', text: '❌ ASSERTION FAILED', ...meta });
+    addOutput({ type: 'error', text: `Assertion: ${assertionPrompt}`, ...meta });
+    addOutput({ type: 'error', text: `Details: ${details}`, ...meta });
     if (isHeadlessMode) {
         // Headless mode: exit with error code
         if (context?.exit) {
@@ -69,7 +80,18 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
     }
     // Interactive mode: caller should clear remaining instructions
 }
-export function handleAssertionSuccess(assertionPrompt, context = null) {
+export function handleAssertionSuccess(assertionPrompt, context = null, stepContext = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
-    addOutput({ type: 'success', text: `✓ Assertion passed: ${assertionPrompt}` });
+    addOutput({
+        type: 'success',
+        text: `✓ Assertion passed: ${assertionPrompt}`,
+        eventType: 'assertion_result',
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        payload: {
+            assertion: assertionPrompt,
+            passed: true
+        }
+    });
 }

package/build/src/device/connection.js

@@ -7,8 +7,8 @@
 import { getDeviceBackend, detectPlatform, setCurrentPlatform, getCurrentPlatform } from "./factory.js";
 let currentBackend = null;
 /**
- * Connect to a device (Android emulator or iOS simulator)
- * @param {string} deviceName - AVD name (Android) or Simulator name (iOS)
+ * Connect to a device (Android target ID/serial or iOS simulator name)
+ * @param {string} deviceName - Android device ID/serial or iOS simulator name
  * @param {string} platform - Optional platform override ('android' or 'ios')
  * @returns {Promise<string>} Device ID
  */

package/build/src/device/factory.js

@@ -11,7 +11,7 @@ import * as iosActions from "./ios/actions.js";
 let currentPlatform = null;
 /**
  * Detect platform from device name or environment variable
- * @param {string} deviceName - The device/AVD/simulator name
+ * @param {string} deviceName - The device ID/name/simulator name
  * @returns {string} 'ios' or 'android'
  */
 export function detectPlatform(deviceName) {

package/build/src/device/interface.js

@@ -11,6 +11,9 @@
  * @property {number} scaled_width - Width as seen by the model (after scaling)
  * @property {number} scaled_height - Height as seen by the model (after scaling)
  * @property {number} scale - Scale factor (scaled_width / device_width)
+ * @property {string} [platform] - Device platform ('android' or 'ios')
+ * @property {string} [device_name] - Human-readable device or simulator name
+ * @property {string} [model] - Device model identifier/name
  */
 /**
  * @typedef {Object} ActionContext
@@ -22,7 +25,7 @@
  * Required exports for a device backend:
  *
  * connectToDevice(deviceName: string): Promise<string>
- *   - Connects to or launches the device/emulator/simulator
+ *   - Connects to the selected device/simulator
  *   - Returns a device ID for subsequent operations
  *
  * getDeviceInfo(deviceId: string): Promise<DeviceInfo>
@@ -37,13 +40,14 @@
  * handleModelAction(deviceId: string, action: object, scale: number, context: ActionContext): Promise<void>
  *   - Executes an action from the CUA model
  *   - Supported action types: click, type, scroll, drag, keypress, wait
+ *   - keypress is limited to single mobile-safe keys only; desktop shortcut chords are unsupported
  */
 export const SUPPORTED_ACTIONS = [
     'click', // Tap at (x, y) coordinates
     'type', // Enter text
     'scroll', // Scroll by (scroll_x, scroll_y)
     'drag', // Drag from start to end via path
-    'keypress', // Press hardware keys (ESC/ESCAPE maps to home)
+    'keypress', // Press a single mobile-safe key (ESC/ESCAPE maps to home)
     'wait', // Wait for UI to settle
     'screenshot' // Capture screen (handled by engine, not backend)
 ];