@loadmill/droid-cua 1.0.0

package/build/src/core/prompts.js

@@ -0,0 +1,158 @@
+/**
+ * System prompt templates for different modes
+ */
+export function buildBaseSystemPrompt(deviceInfo) {
+    return `
+  You are controlling an Android phone in a sandboxed testing environment.
+  Follow the user's instructions to interact with the device.
+
+  The device screen has been scaled down for display.
+  You can interact with any part of the visible phone screen, including system UI, browser UI, and app content.
+
+  The screen you see is ${deviceInfo.scaled_width} x ${deviceInfo.scaled_height} pixels.
+  Pixel (0,0) is at the top-left corner.
+
+  When aiming for visual targets:
+  - Reason carefully about the approximate pixel position.
+  - Click precisely based on your visual estimate.
+
+  Available actions: click, scroll, type, keypress, wait, screenshot.
+
+  CRITICAL - Automatic Timing:
+  - After EVERY action (click, type, keypress, scroll), there is an automatic 500ms delay
+  - This 500ms is sufficient for normal UI updates and animations
+  - DO NOT add 'wait' actions unnecessarily - trust the automatic delay
+
+  Use explicit 'wait' action ONLY in these specific cases:
+  1. After launching apps from home screen or app drawer
+  2. After pressing ENTER that triggers navigation (search, URL, form submit)
+  3. After clicking links that open new apps or pages
+  4. After actions that trigger heavy loading (camera, maps, etc.)
+
+  When you MUST wait:
+  - Click app icon from home → wait → Continue
+  - Type in search box → Press ENTER → wait → Continue
+  - Click link that opens new page/app → wait → Continue
+  - Open camera/maps/heavy feature → wait → Continue
+
+  When you should NOT wait (automatic 500ms handles it):
+  - Clicking UI buttons within a running app (click button - no wait needed)
+  - Typing in text fields (type text - no wait needed)
+  - Scrolling (scroll - no wait needed)
+  - Clicking tabs or menu items within an app (click - no wait needed)
+
+  Rule of thumb: Wait for app launches and navigation. Everything else has automatic timing.
+
+  Perform the user's requested actions within the current view.
+
+  If unsure about visual elements, take a screenshot to improve your reasoning.
+  If unsure about the user's intent, make the best decision you can based on context and continue automatically.
+
+  CRITICAL - Never Ask Questions:
+  - NEVER ask the user for confirmation, clarification, or next steps
+  - NEVER ask questions like "Should I...", "Would you like...", "Do you want me to..."
+  - NEVER wait for user guidance - make autonomous decisions
+  - If stuck, try alternative approaches (go back, try different UI element, restart app)
+  - ONLY stop when the task is complete or you've exhausted reasonable approaches
+
+  Act decisively to complete the task.
+
+  Stop acting once the task appears complete.
+  Only complete the current instruction. Do not proceed beyond the current step unless asked.
+
+  Mobile-Specific Notes:
+  - ESC key maps to the Home button (return to home screen)
+  - Use Home button (ESC) to escape from stuck situations and restart
+  - Back button navigates within apps
+  `;
+}
+export function buildDesignModePrompt(deviceInfo) {
+    const basePrompt = buildBaseSystemPrompt(deviceInfo);
+    return `${basePrompt}
+
+DESIGN MODE:
+You are helping design a test script for an Android app.
+
+Your task:
+1. Understand what the user wants to test from their initial instruction
+2. Explore the app autonomously to understand the flows
+3. Take screenshots and interact as needed to discover the UI and behavior
+4. Once you've successfully completed the user's requested flow, immediately generate the test script
+
+CRITICAL - After Completing the Task:
+- DO NOT navigate back or away from the final screen
+- The final screen state is what matters for verification
+- Generate the test script immediately showing the current state
+- Use assertions to verify state, not navigation
+- "Check that it changed" means verify the current visual state, not navigate elsewhere
+
+CRITICAL - Recognizing When You Are Stuck:
+If you find yourself:
+- Repeating similar actions multiple times (e.g., opening/closing the same app repeatedly)
+- Not reaching a new screen or state after several attempts
+- Unsure about a higher-level decision (which tab to use, which mode to enter, where to start)
+- Unable to find the UI element or feature the user mentioned
+
+THEN STOP ACTING IMMEDIATELY and ask the user for guidance:
+1. Briefly describe what you see on screen now
+2. Explain what you were trying to do and why you're stuck
+3. Ask a single, concrete question to unblock the next step
+
+Example:
+"Chrome is open but I don't see a search bar or new tab button. Should I open a new tab, or is there a specific way you'd like me to navigate?"
+
+DO NOT continue brute-forcing the UI when stuck. The user prefers being asked over watching repeated failed attempts.
+DO NOT ask if the user wants a script after successfully completing the flow - just generate it automatically.
+
+CRITICAL - Test Script Format Rules:
+- One simple instruction per line (NO numbers, NO bullets)
+- Use imperative commands: "Open X", "Click Y", "Type Z"
+- Include "assert: <condition>" lines to validate expected behavior
+- End with "exit"
+- Keep it simple and executable
+
+CORRECT Example:
+\`\`\`
+Open Calculator app
+assert: Calculator app is visible
+Type "2"
+Click the plus button
+Type "3"
+Click the equals button
+assert: result shows 5
+exit
+\`\`\`
+
+WRONG Example (DON'T DO THIS):
+\`\`\`
+1. Open Calculator app
+2. Verify the app opened
+3. etc...
+\`\`\`
+
+Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.
+`;
+}
+export function buildExecutionModePrompt(deviceInfo) {
+    const basePrompt = buildBaseSystemPrompt(deviceInfo);
+    return `${basePrompt}
+
+EXECUTION MODE - Critical Behavior:
+You are executing test script commands one at a time. This is NOT a conversation.
+
+CRITICAL RULES:
+- DO NOT generate conversational text or narration
+- DO NOT ask questions like "What should I do next?", "Would you like...", "Can I assist...?"
+- DO NOT describe what you see on screen
+- DO NOT say "Let me know if you need help" or similar phrases
+- Just execute the action silently and stop immediately
+- Only generate text if the action FAILED or cannot be completed
+
+Your process:
+1. Read the instruction
+2. Execute the required actions
+3. Stop immediately - no commentary, no questions
+
+Each instruction is independent. Do not reference previous instructions or ask about next steps.
+`;
+}

package/build/src/core/session.js

@@ -0,0 +1,57 @@
+/**
+ * Session manages the state for a single CLI session
+ * Includes device info, message history, transcript, and response chaining
+ */
+export class Session {
+    constructor(deviceId, deviceInfo) {
+        this.deviceId = deviceId;
+        this.deviceInfo = deviceInfo;
+        this.messages = [];
+        this.previousResponseId = null;
+        this.transcript = [];
+        this.systemPrompt = null;
+    }
+    /**
+     * Add a message to the conversation history
+     */
+    addMessage(role, content) {
+        this.messages.push({ role, content });
+    }
+    /**
+     * Clear all messages (used in execution mode between turns)
+     * Preserves the system prompt if one was set
+     */
+    clearMessages() {
+        if (this.systemPrompt) {
+            this.messages = [{ role: "system", content: this.systemPrompt }];
+        }
+        else {
+            this.messages = [];
+        }
+    }
+    /**
+     * Set the system prompt and initialize messages array
+     */
+    setSystemPrompt(prompt) {
+        this.systemPrompt = prompt;
+        this.messages = [{ role: "system", content: prompt }];
+    }
+    /**
+     * Add a line to the transcript (for error recovery and logging)
+     */
+    addToTranscript(line) {
+        this.transcript.push(line);
+    }
+    /**
+     * Get the full transcript as a string
+     */
+    getTranscriptText() {
+        return this.transcript.join("\n");
+    }
+    /**
+     * Update the previous response ID for chaining
+     */
+    updateResponseId(responseId) {
+        this.previousResponseId = responseId;
+    }
+}

package/build/src/device/actions.js

@@ -0,0 +1,81 @@
+import { exec } from "child_process";
+import { promisify } from "util";
+import { logger } from "../utils/logger.js";
+const execAsync = promisify(exec);
+function adbShell(deviceId, command) {
+    return execAsync(`adb -s ${deviceId} shell "${command}"`);
+}
+export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    try {
+        const { x, y, x1, y1, x2, y2, text, keys, path } = action;
+        switch (action.type) {
+            case "click":
+                const realX = Math.round(x / scale);
+                const realY = Math.round(y / scale);
+                addOutput({ type: 'action', text: `Clicking at (${realX}, ${realY})` });
+                await adbShell(deviceId, `input tap ${realX} ${realY}`);
+                break;
+            case "scroll":
+                const scrollX = Math.round(action.scroll_x / scale);
+                const scrollY = Math.round(action.scroll_y / scale);
+                addOutput({ type: 'action', text: `Scrolling by (${scrollX}, ${scrollY})` });
+                const startX = 500;
+                const startY = 500;
+                const endX = startX + scrollX;
+                const endY = startY - scrollY; // <--- INVERT Y
+                await adbShell(deviceId, `input swipe ${startX} ${startY} ${endX} ${endY} 500`);
+                break;
+            case "drag":
+                if (path && path.length >= 2) {
+                    const start = path[0];
+                    const end = path[path.length - 1];
+                    const realStartX = Math.round(start.x / scale);
+                    const realStartY = Math.round(start.y / scale);
+                    const realEndX = Math.round(end.x / scale);
+                    const realEndY = Math.round(end.y / scale);
+                    addOutput({ type: 'action', text: `Dragging from (${realStartX}, ${realStartY}) to (${realEndX}, ${realEndY})` });
+                    await adbShell(deviceId, `input swipe ${realStartX} ${realStartY} ${realEndX} ${realEndY} 500`);
+                }
+                else {
+                    addOutput({ type: 'info', text: `Drag action missing valid path: ${JSON.stringify(action)}` });
+                }
+                break;
+            case "type":
+                addOutput({ type: 'action', text: `Typing text: ${text}` });
+                const escapedText = text.replace(/(["\\$`])/g, "\\$1").replace(/ /g, "%s");
+                await adbShell(deviceId, `input text "${escapedText}"`);
+                break;
+            case "keypress":
+                // Map ESC to Android Home button (since ESC doesn't exist on mobile)
+                const mappedKeys = keys.map(key => {
+                    if (key.toUpperCase() === 'ESC' || key.toUpperCase() === 'ESCAPE') {
+                        return 'KEYCODE_HOME';
+                    }
+                    return key;
+                });
+                addOutput({ type: 'action', text: `Pressing key: ${mappedKeys.join(', ')}` });
+                for (const key of mappedKeys) {
+                    await adbShell(deviceId, `input keyevent ${key}`);
+                }
+                break;
+            case "wait":
+                addOutput({ type: 'action', text: 'Waiting...' });
+                await new Promise(res => setTimeout(res, 1000));
+                break;
+            default:
+                addOutput({ type: 'info', text: `Unknown action: ${JSON.stringify(action)}` });
+        }
+    }
+    catch (error) {
+        // Log full error details to file
+        logger.error('Action execution error', {
+            action,
+            message: error.message,
+            stack: error.stack
+        });
+        // Show user-friendly error message
+        addOutput({ type: 'error', text: `Error executing action: ${error.message}` });
+        addOutput({ type: 'info', text: 'Full error details have been logged to the debug log.' });
+    }
+}

package/build/src/device/assertions.js

@@ -0,0 +1,75 @@
+/**
+ * Assertion handling for script validation
+ */
+export function isAssertion(userInput) {
+    const trimmed = userInput.trim();
+    const lower = trimmed.toLowerCase();
+    return lower.startsWith("assert:") || lower.startsWith("assert ");
+}
+export function extractAssertionPrompt(userInput) {
+    const trimmed = userInput.trim();
+    const lower = trimmed.toLowerCase();
+    // Handle "assert:" or "Assert:"
+    if (lower.startsWith("assert:")) {
+        return trimmed.substring("assert:".length).trim();
+    }
+    // Handle "assert " or "Assert "
+    if (lower.startsWith("assert ")) {
+        return trimmed.substring("assert".length).trim();
+    }
+    return trimmed;
+}
+export function buildAssertionSystemPrompt(baseSystemPrompt, assertionPrompt) {
+    return `${baseSystemPrompt}
+
+ASSERTION MODE:
+You are now validating an assertion. The user has provided an assertion statement that you must verify.
+
+Your task:
+1. Take screenshots and perform LIMITED actions if needed to validate the assertion.
+2. Determine if the assertion is TRUE or FALSE based on the current state.
+3. You MUST respond with a clear verdict in this exact format:
+   - If the assertion is true, include the text: "ASSERTION RESULT: PASS"
+   - If the assertion is false or cannot be confidently validated, include: "ASSERTION RESULT: FAIL"
+4. After the verdict, provide a brief explanation (1-2 sentences) of why it passed or failed.
+
+The assertion to validate is: "${assertionPrompt}"
+
+Remember:
+- If you cannot confidently validate the assertion, treat it as FAIL.
+- You must include either "ASSERTION RESULT: PASS" or "ASSERTION RESULT: FAIL" in your response.
+- Be thorough but efficient. Only take the actions necessary to validate the assertion.`;
+}
+export function checkAssertionResult(transcript) {
+    const transcriptText = transcript.join("\n");
+    const hasPassed = transcriptText.includes("ASSERTION RESULT: PASS");
+    const hasFailed = transcriptText.includes("ASSERTION RESULT: FAIL");
+    return {
+        passed: hasPassed && !hasFailed,
+        failed: hasFailed || !hasPassed,
+    };
+}
+export function extractFailureDetails(transcript) {
+    const recentTranscript = transcript.slice(-5).join("\n");
+    const parts = recentTranscript.split("ASSERTION RESULT: FAIL");
+    return parts[1]?.trim() || "Could not confidently validate the assertion.";
+}
+export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context) {
+    const details = extractFailureDetails(transcript);
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    addOutput({ type: 'error', text: '❌ ASSERTION FAILED' });
+    addOutput({ type: 'error', text: `Assertion: ${assertionPrompt}` });
+    addOutput({ type: 'error', text: `Details: ${details}` });
+    if (isHeadlessMode) {
+        // Headless mode: exit with error code
+        if (context?.exit) {
+            context.exit();
+        }
+        process.exit(1);
+    }
+    // Interactive mode: caller should clear remaining instructions
+}
+export function handleAssertionSuccess(assertionPrompt, context = null) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    addOutput({ type: 'success', text: `✓ Assertion passed: ${assertionPrompt}` });
+}

package/build/src/device/connection.js

@@ -0,0 +1,123 @@
+import { exec, spawn } from "child_process";
+import { once } from "events";
+import { promisify } from "util";
+import sharp from "sharp";
+import { logger } from "../utils/logger.js";
+const execAsync = promisify(exec);
+function wait(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+async function listConnectedDevices() {
+    const { stdout } = await execAsync("adb devices");
+    return stdout
+        .trim()
+        .split("\n")
+        .slice(1)
+        .map(line => line.split("\t")[0])
+        .filter(id => id.length > 0);
+}
+async function waitForDeviceConnection(avdName, timeoutMs = 120000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const devices = await listConnectedDevices();
+        const match = devices.find(id => id.includes(avdName));
+        if (match)
+            return match;
+        await wait(2000);
+    }
+    return null;
+}
+async function waitForDeviceBoot(deviceId, timeoutMs = 60000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        try {
+            const { stdout } = await execAsync(`adb -s ${deviceId} shell getprop sys.boot_completed`);
+            if (stdout.trim() === "1")
+                return true;
+        }
+        catch { }
+        await wait(2000);
+    }
+    return false;
+}
+export async function connectToDevice(avdName) {
+    const devices = await listConnectedDevices();
+    for (const id of devices) {
+        if (id.startsWith("emulator-")) {
+            try {
+                const { stdout } = await execAsync(`adb -s ${id} emu avd name`);
+                if (stdout.trim() === avdName) {
+                    console.log(`Emulator ${avdName} is already running as ${id}`);
+                    return id;
+                }
+            }
+            catch { }
+        }
+    }
+    console.log(`No emulator with AVD "${avdName}" is running. Launching...`);
+    const emulatorProcess = spawn("emulator", ["-avd", avdName], { detached: true, stdio: "ignore" });
+    emulatorProcess.unref();
+    const deviceId = await waitForDeviceConnection("emulator-", 120000);
+    if (!deviceId) {
+        console.error(`Emulator ${avdName} did not appear in time.`);
+        process.exit(1);
+    }
+    console.log(`Device ${deviceId} detected. Waiting for boot...`);
+    const booted = await waitForDeviceBoot(deviceId);
+    if (!booted) {
+        console.error(`Emulator ${avdName} did not finish booting.`);
+        process.exit(1);
+    }
+    console.log(`Emulator ${avdName} is fully booted.`);
+    return deviceId;
+}
+export async function getDeviceInfo(deviceId) {
+    const { stdout } = await execAsync(`adb -s ${deviceId} shell wm size`);
+    const match = stdout.match(/Physical size:\s*(\d+)x(\d+)/);
+    if (!match) {
+        console.error("Could not get device screen size.");
+        process.exit(1);
+    }
+    const [_, width, height] = match.map(Number);
+    const targetWidth = 400;
+    const scale = width > targetWidth ? targetWidth / width : 1.0;
+    const scaledWidth = Math.round(width * scale);
+    const scaledHeight = Math.round(height * scale);
+    return {
+        device_width: width,
+        device_height: height,
+        scaled_width: scaledWidth,
+        scaled_height: scaledHeight,
+        scale,
+    };
+}
+export async function getScreenshotAsBase64(deviceId, deviceInfo) {
+    const adb = spawn("adb", ["-s", deviceId, "exec-out", "screencap", "-p"]);
+    const chunks = [];
+    const stderrChunks = [];
+    adb.stdout.on("data", chunk => chunks.push(chunk));
+    adb.stderr.on("data", err => {
+        stderrChunks.push(err);
+        console.error("ADB stderr:", err.toString());
+    });
+    const [code] = await once(adb, "close");
+    if (code !== 0) {
+        const stderrOutput = Buffer.concat(stderrChunks).toString();
+        logger.error(`ADB screencap failed with code ${code}`, { stderr: stderrOutput });
+        throw new Error(`adb screencap exited with code ${code}`);
+    }
+    let buffer = Buffer.concat(chunks);
+    logger.debug(`Screenshot captured: ${buffer.length} bytes before scaling`);
+    if (buffer.length === 0) {
+        logger.error('Screenshot buffer is empty!', { deviceId, chunks: chunks.length });
+        throw new Error('Screenshot capture returned empty buffer');
+    }
+    if (deviceInfo.scale < 1.0) {
+        buffer = await sharp(buffer)
+            .resize({ width: deviceInfo.scaled_width, height: deviceInfo.scaled_height })
+            .png()
+            .toBuffer();
+        logger.debug(`Screenshot scaled: ${buffer.length} bytes after scaling`);
+    }
+    return buffer.toString("base64");
+}

package/build/src/device/openai.js

@@ -0,0 +1,124 @@
+import OpenAI from "openai";
+import dotenv from "dotenv";
+import { logger } from "../utils/logger.js";
+dotenv.config();
+const openai = new OpenAI({
+    apiKey: process.env.OPENAI_API_KEY,
+});
+/**
+ * Revise a test script based on user feedback using simple chat completion
+ * @param {string} originalScript - The original test script
+ * @param {string} revisionRequest - User's requested changes
+ * @returns {Promise<string>} - The revised test script
+ */
+export async function reviseTestScript(originalScript, revisionRequest) {
+    const response = await openai.chat.completions.create({
+        model: "gpt-4o",
+        messages: [{
+                role: "system",
+                content: `You are editing a test script based on user feedback.
+
+Current test script:
+${originalScript}
+
+User's revision request:
+${revisionRequest}
+
+Apply the user's changes and output the revised test script.
+
+FORMAT RULES:
+- One simple instruction per line (NO numbers, NO bullets)
+- Use imperative commands: "Open X", "Click Y", "Type Z"
+- Include "assert: <condition>" lines to validate expected behavior
+- End with "exit"
+
+Output only the revised test script, nothing else.`
+            }]
+    });
+    return response.choices[0].message.content.trim();
+}
+export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, }) {
+    const input = [...messages];
+    if (callId && screenshotBase64) {
+        input.push({
+            type: "computer_call_output",
+            call_id: callId,
+            output: {
+                type: "computer_screenshot",
+                image_url: `data:image/png;base64,${screenshotBase64}`,
+            },
+        });
+    }
+    const requestParams = {
+        model: "computer-use-preview",
+        previous_response_id: previousResponseId || undefined,
+        tools: [{
+                type: "computer_use_preview",
+                display_width: deviceInfo.scaled_width,
+                display_height: deviceInfo.scaled_height,
+                environment: "browser",
+            }],
+        input,
+        store: true,
+        reasoning: { generate_summary: "concise" },
+        truncation: "auto",
+    };
+    // Log request details (without full screenshot to avoid clutter)
+    const requestLog = {
+        ...requestParams,
+        input: input.map(item => {
+            if (item.type === "computer_call_output" && item.output?.image_url) {
+                // Extract actual base64 length from the image_url
+                const imageUrl = item.output.image_url;
+                const base64Data = imageUrl.replace('data:image/png;base64,', '');
+                return {
+                    ...item,
+                    output: {
+                        ...item.output,
+                        image_url: `data:image/png;base64,[${base64Data.length} chars]`
+                    },
+                    current_url: item.current_url,
+                    acknowledged_safety_checks: item.acknowledged_safety_checks
+                };
+            }
+            return item;
+        })
+    };
+    logger.debug('CUA Request:', requestLog);
+    try {
+        const response = await openai.responses.create(requestParams);
+        // Log ALL output item types to catch everything
+        const outputTypes = (response.output || []).map(item => item.type);
+        const toolCalls = (response.output || [])
+            .filter(item => item.type === 'computer_call')
+            .map(item => ({
+            call_id: item.call_id,
+            action_type: item.action?.type
+        }));
+        const safetyChecks = (response.output || [])
+            .filter(item => item.type === 'pending_safety_check')
+            .map(item => ({
+            id: item.id,
+            code: item.code
+        }));
+        // Log full output array if there are unaccounted items
+        const accountedItems = toolCalls.length + safetyChecks.length;
+        const totalItems = response.output?.length || 0;
+        logger.debug('CUA Response:', {
+            id: response.id,
+            output_length: totalItems,
+            output_types: outputTypes,
+            tool_calls: toolCalls.length > 0 ? toolCalls : 'none',
+            pending_safety_checks: safetyChecks.length > 0 ? safetyChecks : 'none'
+        });
+        // If we're missing items in our logging, log the full output for investigation
+        if (accountedItems < totalItems) {
+            logger.debug('UNACCOUNTED OUTPUT ITEMS - Full output array:', response.output);
+        }
+        return response;
+    }
+    catch (err) {
+        logger.error('CUA Request failed', { request: requestLog, error: err });
+        throw err;
+    }
+}