@loadmill/droid-cua 1.1.1 → 2.0.0

package/build/src/cli/device-selector.js

@@ -51,49 +51,76 @@ function renderSelection(title, items) {
     });
 }
 /**
- * Get list of available Android AVDs
+ * Get a unified Android list: connected ADB devices + launchable AVDs
  */
 async function getAndroidDevices() {
-    const devices = [];
-    // Get running emulators
+    const connectedDevices = [];
+    const availableAvds = [];
+    const avdInventory = new Set();
+    const connectedAvdNames = new Set();
     try {
         const { stdout: adbOutput } = await execAsync("adb devices");
-        const runningIds = adbOutput
-            .trim()
+        const connectedIds = adbOutput
             .split("\n")
             .slice(1)
-            .map((line) => line.split("\t")[0])
-            .filter((id) => id.startsWith("emulator-"));
-        for (const id of runningIds) {
+            .map((line) => line.trim())
+            .filter(Boolean)
+            .map((line) => {
+            const [id, state] = line.split("\t");
+            return { id: id?.trim() || "", state: state?.trim() || "" };
+        })
+            .filter((entry) => entry.id.length > 0 && entry.state === "device");
+        for (const entry of connectedIds) {
+            const id = entry.id;
+            let model = "";
+            let avdName = "";
             try {
-                const { stdout } = await execAsync(`adb -s ${id} emu avd name`);
-                const name = stdout.trim();
-                devices.push({
-                    label: `${name} (running)`,
-                    value: name,
-                    running: true,
-                });
+                const { stdout } = await execAsync(`adb -s "${id}" shell getprop ro.product.model`);
+                model = stdout.trim();
             }
             catch { }
+            if (id.startsWith("emulator-")) {
+                try {
+                    const { stdout } = await execAsync(`adb -s "${id}" emu avd name`);
+                    avdName = stdout.trim();
+                }
+                catch { }
+            }
+            if (avdName) {
+                connectedAvdNames.add(avdName);
+            }
+            connectedDevices.push({
+                label: model ? `${model} (${id})` : id,
+                value: `adb:${id}`,
+                running: true,
+            });
         }
     }
     catch { }
-    // Get available AVDs
     try {
         const { stdout } = await execAsync("emulator -list-avds");
-        const avds = stdout.trim().split("\n").filter((name) => name.length > 0);
+        const avds = stdout
+            .split("\n")
+            .map((line) => line.trim())
+            .filter(Boolean);
         for (const avd of avds) {
-            if (!devices.some((d) => d.value === avd)) {
-                devices.push({
-                    label: avd,
-                    value: avd,
-                    running: false,
-                });
-            }
+            avdInventory.add(avd);
         }
     }
     catch { }
-    return devices;
+    for (const connectedAvdName of connectedAvdNames) {
+        avdInventory.delete(connectedAvdName);
+    }
+    for (const avdName of avdInventory) {
+        availableAvds.push({
+            label: avdName,
+            value: `avd:${avdName}`,
+            running: false,
+        });
+    }
+    connectedDevices.sort((a, b) => a.label.localeCompare(b.label));
+    availableAvds.sort((a, b) => a.label.localeCompare(b.label));
+    return [...connectedDevices, ...availableAvds];
 }
 /**
  * Get list of available iOS Simulators (iPhones only)
@@ -147,7 +174,7 @@ export async function selectDevice() {
     const hasIOS = iosDevices.length > 0;
     if (!hasAndroid && !hasIOS) {
         console.error("\nNo devices found!");
-        console.error("  Android: Create an AVD with Android Studio");
+        console.error("  Android: Connect a device or create an AVD, then run droid-cua again");
         console.error("  iOS: Xcode Simulator must be available");
         process.exit(1);
     }
@@ -156,7 +183,7 @@ export async function selectDevice() {
     if (hasAndroid) {
         const runningCount = androidDevices.filter((d) => d.running).length;
         platformOptions.push({
-            label: `Android${runningCount > 0 ? ` (${runningCount} running)` : ""} - ${androidDevices.length} emulator(s)`,
+            label: `Android${runningCount > 0 ? ` (${runningCount} connected)` : ""} - ${androidDevices.length} target(s)`,
             value: "android",
         });
     }
@@ -179,7 +206,7 @@ export async function selectDevice() {
     }
     // Select device
     const deviceList = platform === "ios" ? iosDevices : androidDevices;
-    const deviceType = platform === "ios" ? "Simulator" : "Emulator";
+    const deviceType = platform === "ios" ? "Simulator" : "Android Target";
     let deviceName;
     if (deviceList.length === 1) {
         deviceName = deviceList[0].value;

package/build/src/commands/help.js

@@ -16,7 +16,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  droid-cua --avd <device-name> [options]' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Options:' });
-    addOutput({ type: 'info', text: '  --avd <name>           Device name (Android AVD or iOS Simulator)' });
+    addOutput({ type: 'info', text: '  --avd <name>           Device name (Android device ID/serial or iOS Simulator)' });
     addOutput({ type: 'info', text: '  --platform <platform>  Force platform: android or ios' });
     addOutput({ type: 'info', text: '  --instructions <file>  Run test file in headless mode' });
     addOutput({ type: 'info', text: '  --record               Record screenshots during execution' });
@@ -37,7 +37,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  /loadmill <command>      Run Loadmill test flows using natural language' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Platform Support:' });
-    addOutput({ type: 'info', text: '  Android: Uses ADB to communicate with Android emulators' });
+    addOutput({ type: 'info', text: '  Android: Uses ADB to communicate with Android devices (physical or emulator)' });
     addOutput({ type: 'info', text: '  iOS:     Uses Appium + XCUITest for iOS Simulator automation' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Platform Detection:' });
@@ -53,7 +53,8 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  Note: Appium server is auto-started when iOS platform is detected' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Examples:' });
-    addOutput({ type: 'info', text: '  droid-cua --avd Pixel_8_API_35          (Android emulator)' });
+    addOutput({ type: 'info', text: '  droid-cua --avd adb:emulator-5554        (Connected Android target by adb serial)' });
+    addOutput({ type: 'info', text: '  droid-cua --avd avd:Pixel_8_API_35       (Launch Android AVD then connect)' });
     addOutput({ type: 'info', text: '  droid-cua --avd "iPhone 16"             (iOS Simulator, auto-detected)' });
     addOutput({ type: 'info', text: '  droid-cua --platform ios --avd MySim    (Force iOS platform)' });
     addOutput({ type: 'info', text: '  /create login-test                      (design a new test)' });

package/build/src/core/execution-engine.js

@@ -3,6 +3,32 @@ import { writeFile } from "fs/promises";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
 import { handleModelAction } from "../device/actions.js";
 import { sendCUARequest } from "../device/openai.js";
+import { emitDesktopDebug } from "../utils/desktop-debug.js";
+function extractComputerCalls(items) {
+    const entries = [];
+    for (const item of items) {
+        if (item?.type !== "computer_call")
+            continue;
+        const baseCallId = item.call_id || item.id;
+        if (!baseCallId)
+            continue;
+        const actions = [];
+        if (Array.isArray(item.actions)) {
+            for (const actionEntry of item.actions) {
+                if (!actionEntry)
+                    continue;
+                const action = actionEntry.action || actionEntry;
+                if (action?.type)
+                    actions.push(action);
+            }
+        }
+        else if (item.action?.type) {
+            actions.push(item.action);
+        }
+        entries.push({ call_id: baseCallId, actions });
+    }
+    return entries;
+}
 export class ExecutionEngine {
     constructor(session, options = {}) {
         this.session = session;
@@ -17,9 +43,15 @@ export class ExecutionEngine {
      * @param {Function} trackAction - Optional callback to track actions for stuck detection
      * @param {Object} context - Optional Ink context for output
      */
-    async runFullTurn(response, trackAction = null, context = null) {
+    async runFullTurn(response, trackAction = null, context = null, stepContext = null) {
         const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
         let newResponseId = response.id;
+        const eventMeta = (extra = {}) => ({
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+            ...extra
+        });
         while (true) {
             // Check for interruption before processing next batch of actions
             if (trackAction) {
@@ -29,7 +61,7 @@ export class ExecutionEngine {
                 }
             }
             const items = response.output || [];
-            const actions = items.filter(item => item.type === "computer_call");
+            const computerCalls = extractComputerCalls(items);
             // ── Collect pending safety checks ──
             const pendingSafetyChecks = items
                 .filter(item => item.type === "pending_safety_check")
@@ -39,7 +71,12 @@ export class ExecutionEngine {
                 if (item.type === "reasoning") {
                     for (const entry of item.summary) {
                         if (entry.type === "summary_text") {
-                            addOutput({ type: 'reasoning', text: entry.text });
+                            addOutput({
+                                type: 'reasoning',
+                                text: entry.text,
+                                eventType: 'reasoning',
+                                ...eventMeta()
+                            });
                             this.session.addToTranscript(`[Reasoning] ${entry.text}`);
                         }
                     }
@@ -47,45 +84,94 @@ export class ExecutionEngine {
                 else if (item.type === "message") {
                     const textPart = item.content.find(c => c.type === "output_text");
                     if (textPart) {
-                        addOutput({ type: 'assistant', text: textPart.text });
+                        addOutput({
+                            type: 'assistant',
+                            text: textPart.text,
+                            eventType: 'assistant_message',
+                            ...eventMeta()
+                        });
                         this.session.addToTranscript(`[Assistant] ${textPart.text}`);
                     }
                 }
                 else if (item.type === "pending_safety_check") {
-                    addOutput({ type: 'warning', text: `⚠️ Safety check: ${item.code} - ${item.message}` });
+                    addOutput({
+                        type: 'warning',
+                        text: `⚠️ Safety check: ${item.code} - ${item.message}`,
+                        eventType: 'system_message',
+                        ...eventMeta({
+                            payload: {
+                                id: item.id,
+                                code: item.code,
+                                message: item.message
+                            }
+                        })
+                    });
                 }
             }
-            if (actions.length === 0) {
+            if (computerCalls.length === 0) {
                 // No actions = turn complete
                 break;
             }
             // ── Process model actions ──
-            for (const { action, call_id } of actions) {
-                if (action.type === "screenshot") {
-                    addOutput({ type: 'info', text: '📸 Capturing screen' });
-                }
-                else {
-                    await handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, context);
-                    // Track action and check for interruption
-                    if (trackAction) {
-                        const shouldStop = trackAction(action);
-                        if (shouldStop) {
-                            // User interrupted - stop execution immediately
-                            return newResponseId;
-                        }
+            for (const { call_id, actions } of computerCalls) {
+                if (!call_id)
+                    continue;
+                let sawExplicitScreenshotAction = false;
+                for (const action of actions) {
+                    if (action.type === "screenshot") {
+                        sawExplicitScreenshotAction = true;
+                        addOutput({
+                            type: 'info',
+                            text: 'Capturing screen',
+                            eventType: 'screenshot_captured',
+                            actionType: 'screenshot',
+                            ...eventMeta({
+                                payload: {
+                                    callId: call_id,
+                                    source: 'explicit_action'
+                                }
+                            })
+                        });
                     }
-                    // Add delay after UI-changing actions to let the interface update
-                    // before taking the screenshot (except for explicit wait actions which have their own delay)
-                    if (action.type !== "wait") {
-                        await new Promise(resolve => setTimeout(resolve, 500));
+                    else {
+                        await handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
+                            ...context,
+                            stepId: stepContext?.stepId,
+                            instructionIndex: stepContext?.instructionIndex
+                        });
+                        // Track action and check for interruption
+                        if (trackAction) {
+                            const shouldStop = trackAction(action);
+                            if (shouldStop) {
+                                // User interrupted - stop execution immediately
+                                return newResponseId;
+                            }
+                        }
+                        // Add delay after UI-changing actions to let the interface update
+                        // before taking the screenshot (except for explicit wait actions which have their own delay)
+                        if (action.type !== "wait") {
+                            await new Promise(resolve => setTimeout(resolve, 500));
+                        }
                     }
                 }
                 const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+                emitDesktopDebug("device.screenshot", "device", {
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }, {
+                    source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+                    callId: call_id,
+                    width: this.session.deviceInfo?.scaled_width,
+                    height: this.session.deviceInfo?.scaled_height,
+                    base64Length: screenshotBase64.length
+                });
                 if (this.recordScreenshots && this.screenshotDir) {
                     const framePath = path.join(this.screenshotDir, `frame_${String(Date.now())}.png`);
                     await writeFile(framePath, Buffer.from(screenshotBase64, "base64"));
                 }
                 // Build next input: screenshot + any carryover reasoning
+                const selectedCuaModel = process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
                 const input = [{
                         type: "computer_call_output",
                         call_id,
@@ -93,21 +179,37 @@ export class ExecutionEngine {
                             type: "computer_screenshot",
                             image_url: `data:image/png;base64,${screenshotBase64}`,
                         },
-                        current_url: getCurrentPlatform() === "ios" ? "ios://simulator" : "android://emulator",
+                        ...(selectedCuaModel === "computer-use-preview"
+                            ? { current_url: getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
+                            : {}),
                         ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
                     }];
                 response = await sendCUARequest({
                     messages: input,
                     previousResponseId: newResponseId,
                     deviceInfo: this.session.deviceInfo,
+                    debugContext: {
+                        scope: context?.sessionId ? "design" : "execution",
+                        runId: context?.runId,
+                        sessionId: context?.sessionId,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex
+                    }
                 });
                 newResponseId = response.id;
+                // Each tool output advances the response chain; process newly returned calls next.
+                break;
             }
         }
         // ── At end, if last output was only reasoning ──
         const finalItems = response.output || [];
         if (finalItems.length > 0 && finalItems.at(-1).type === "reasoning") {
-            addOutput({ type: 'info', text: 'Warning: last item was reasoning without follow-up. Dropping to avoid 400 error.' });
+            addOutput({
+                type: 'info',
+                text: 'Warning: last item was reasoning without follow-up. Dropping to avoid 400 error.',
+                eventType: 'system_message',
+                ...eventMeta()
+            });
         }
         return newResponseId;
     }

package/build/src/core/prompts.js

@@ -1,9 +1,32 @@
 /**
  * System prompt templates for different modes
  */
-export function buildBaseSystemPrompt(deviceInfo) {
-    return `
-  You are controlling an Android phone in a sandboxed testing environment.
+function appendCustomSection(prompt, customText) {
+    const trimmed = typeof customText === "string" ? customText.trim() : "";
+    if (!trimmed) {
+        return prompt;
+    }
+    return `${prompt}
+
+CUSTOM INSTRUCTIONS:
+${trimmed}
+`;
+}
+function describeControlledDevice(deviceInfo = {}) {
+    const platform = typeof deviceInfo.platform === "string" ? deviceInfo.platform.trim().toLowerCase() : "";
+    const deviceName = typeof deviceInfo.device_name === "string" ? deviceInfo.device_name.trim() : "";
+    if (platform === "ios") {
+        return deviceName ? `an iOS simulator (${deviceName})` : "an iOS device";
+    }
+    if (platform === "android") {
+        return deviceName ? `an Android device (${deviceName})` : "an Android device";
+    }
+    return "a mobile device";
+}
+export function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
+    const controlledDevice = describeControlledDevice(deviceInfo);
+    const prompt = `
+  You are controlling ${controlledDevice} in a sandboxed testing environment.
   Follow the user's instructions to interact with the device.
 
   The device screen has been scaled down for display.
@@ -18,6 +41,13 @@ export function buildBaseSystemPrompt(deviceInfo) {
 
   Available actions: click, scroll, type, keypress, wait, screenshot.
 
+  CRITICAL - Mobile Input Constraints:
+  - This is a mobile device, not a desktop. Do NOT use desktop keyboard shortcuts or modifier chords.
+  - NEVER emit key combinations such as CTRL+A, CMD+A, CTRL+C, CTRL+V, ALT+TAB, SHIFT+ENTER, or similar shortcuts.
+  - Use 'keypress' only for a single mobile-safe key when absolutely necessary.
+  - To replace text, tap into the field and type the desired value. If correction is needed, use mobile-safe deletion only.
+  - Prefer tapping visible controls over hardware key events.
+
   CRITICAL - Automatic Timing:
   - After EVERY action (click, type, keypress, scroll), there is an automatic 500ms delay
   - This 500ms is sufficient for normal UI updates and animations
@@ -63,15 +93,22 @@ export function buildBaseSystemPrompt(deviceInfo) {
   Mobile-Specific Notes:
   - ESC key maps to the Home button (return to home screen)
   - Use Home button (ESC) to escape from stuck situations and restart
-  - Back button navigates within apps
+  - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
   `;
+    return appendCustomSection(prompt, customInstructions.basePromptInstructions);
 }
-export function buildDesignModePrompt(deviceInfo) {
-    const basePrompt = buildBaseSystemPrompt(deviceInfo);
-    return `${basePrompt}
+export function buildDesignModePrompt(deviceInfo, customInstructions = {}) {
+    const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
+    const mergedBaseInstructions = [customInstructions.basePromptInstructions, designCustomText].filter(Boolean).join("\n\n");
+    const basePrompt = buildBaseSystemPrompt(deviceInfo, {
+        ...customInstructions,
+        basePromptInstructions: mergedBaseInstructions
+    });
+    const prompt = `${basePrompt}
 
 DESIGN MODE:
 You are helping design a test script for an Android app.
+Some tests intentionally validate negative outcomes (errors, failures, rejected inputs). These are expected and should be treated as successful progress when they match the test goal.
 
 Your task:
 1. Understand what the user wants to test from their initial instruction
@@ -85,6 +122,7 @@ CRITICAL - After Completing the Task:
 - Generate the test script immediately showing the current state
 - Use assertions to verify state, not navigation
 - "Check that it changed" means verify the current visual state, not navigate elsewhere
+- If the target validation state is visible (including expected error states), STOP actions and immediately output the final test script
 
 CRITICAL - Recognizing When You Are Stuck:
 If you find yourself:
@@ -104,10 +142,22 @@ Example:
 DO NOT continue brute-forcing the UI when stuck. The user prefers being asked over watching repeated failed attempts.
 DO NOT ask if the user wants a script after successfully completing the flow - just generate it automatically.
 
+CRITICAL - Off-Screen Element Discovery:
+- If a required element is not visible, assume it may be off-screen before changing strategy
+- Humans naturally scroll when UI appears cropped; do the same
+- Use this discovery sequence before retries or fallback navigation:
+  1. Scroll the screen in the likely direction to reveal hidden content
+  2. If still missing, do one minimal fallback (e.g., close overlay or go back once), then retry discovery
+- Do not repeat already-successful actions while searching for an off-screen target
+
 CRITICAL - Test Script Format Rules:
 - One simple instruction per line (NO numbers, NO bullets)
 - Use imperative commands: "Open X", "Click Y", "Type Z"
 - Include "assert: <condition>" lines to validate expected behavior
+- Normalize validation wording into assertions:
+  - Convert "check", "verify", "ensure", "fetch", and "compare" intent into explicit "assert: ..." lines
+  - Do not leave standalone "Check ..." or "Verify ..." lines in the final script
+- Merge duplicate or near-duplicate validation lines into one clear assertion
 - End with "exit"
 - Keep it simple and executable
 
@@ -132,10 +182,16 @@ WRONG Example (DON'T DO THIS):
 
 Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.
 `;
+    return prompt;
 }
-export function buildExecutionModePrompt(deviceInfo) {
-    const basePrompt = buildBaseSystemPrompt(deviceInfo);
-    return `${basePrompt}
+export function buildExecutionModePrompt(deviceInfo, customInstructions = {}) {
+    const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
+    const mergedBaseInstructions = [customInstructions.basePromptInstructions, executionCustomText].filter(Boolean).join("\n\n");
+    const basePrompt = buildBaseSystemPrompt(deviceInfo, {
+        ...customInstructions,
+        basePromptInstructions: mergedBaseInstructions
+    });
+    const prompt = `${basePrompt}
 
 EXECUTION MODE - Critical Behavior:
 You are executing test script commands one at a time. This is NOT a conversation.
@@ -147,6 +203,10 @@ CRITICAL RULES:
 - DO NOT say "Let me know if you need help" or similar phrases
 - Just execute the action silently and stop immediately
 - Only generate text if the action FAILED or cannot be completed
+- Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses
+- If target is not visible, perform bounded off-screen discovery first:
+  1. Scroll the screen in the likely direction to reveal hidden controls
+  2. If still missing, do one minimal fallback (e.g., close overlay or go back once), then retry
 
 Your process:
 1. Read the instruction
@@ -155,4 +215,5 @@ Your process:
 
 Each instruction is independent. Do not reference previous instructions or ask about next steps.
 `;
+    return prompt;
 }

package/build/src/device/actions.js

@@ -7,7 +7,7 @@
 import { getDeviceBackend, getCurrentPlatform } from "./factory.js";
 /**
  * Handle an action from the CUA model
- * @param {string} deviceId - The device/emulator/simulator ID
+ * @param {string} deviceId - The connected device/simulator ID
  * @param {object} action - The action to execute
  * @param {number} scale - Scale factor for coordinates
  * @param {object} context - Context with addOutput function