@loadmill/droid-cua 1.0.0

package/build/src/commands/create.js

@@ -0,0 +1,42 @@
+/**
+ * Create command handler
+ */
+import { testExists } from "../test-store/test-manager.js";
+import { DesignModeInk } from "../modes/design-mode-ink.js";
+/**
+ * Handle /create command
+ * @param {string} args - Test name
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context (includes engine, addOutput, etc.)
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleCreate(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const testName = args.trim();
+    // Check if test name provided
+    if (!testName) {
+        addOutput({ type: 'error', text: 'Usage: /create <test-name>' });
+        addOutput({ type: 'info', text: '' });
+        addOutput({ type: 'info', text: 'Example:' });
+        addOutput({ type: 'info', text: '  /create login-flow' });
+        addOutput({ type: 'info', text: '  /create calculator-test' });
+        return true; // Continue loop
+    }
+    // Check if test already exists
+    const exists = await testExists(testName);
+    if (exists) {
+        addOutput({ type: 'error', text: `Test already exists: ${testName}` });
+        addOutput({ type: 'info', text: 'Choose a different name or delete the existing test first.' });
+        return true; // Continue loop
+    }
+    // Create design mode
+    const designMode = new DesignModeInk(session, context.engine, testName, context);
+    // Store reference in context so ink-shell can route inputs to it
+    if (context.setActiveDesignMode) {
+        context.setActiveDesignMode(designMode);
+    }
+    // Start design mode conversation
+    await designMode.start();
+    addOutput({ type: 'system', text: '=== Exited Design Mode ===' });
+    return true; // Continue loop
+}

package/build/src/commands/edit.js

@@ -0,0 +1,61 @@
+/**
+ * Edit command handler
+ */
+import { spawn } from 'child_process';
+import { testExists } from "../test-store/test-manager.js";
+import path from "path";
+import { fileURLToPath } from "url";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const TESTS_DIR = path.join(__dirname, "../../tests");
+/**
+ * Handle /edit command
+ * @param {string} args - Test name
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleEdit(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const testName = args.trim();
+    // Check if test name provided
+    if (!testName) {
+        addOutput({ type: 'error', text: 'Usage: /edit <test-name>' });
+        addOutput({ type: 'info', text: '' });
+        addOutput({ type: 'info', text: 'Example:' });
+        addOutput({ type: 'info', text: '  /edit example' });
+        return true; // Continue loop
+    }
+    // Check if test exists
+    const exists = await testExists(testName);
+    if (!exists) {
+        addOutput({ type: 'error', text: `Test not found: ${testName}` });
+        addOutput({ type: 'info', text: 'Use /list to see available tests.' });
+        return true; // Continue loop
+    }
+    // Determine editor to use
+    const editor = process.env.EDITOR || process.env.VISUAL || 'nano';
+    const filename = testName.endsWith('.dcua') ? testName : `${testName}.dcua`;
+    const filepath = path.join(TESTS_DIR, filename);
+    addOutput({ type: 'system', text: `Opening ${testName} in ${editor}...` });
+    addOutput({ type: 'info', text: 'Save and exit the editor to return to droid-cua.' });
+    // Open editor in foreground (blocking)
+    return new Promise((resolve) => {
+        const editorProcess = spawn(editor, [filepath], {
+            stdio: 'inherit', // Inherit stdin/stdout/stderr to allow interactive editing
+        });
+        editorProcess.on('exit', (code) => {
+            if (code === 0) {
+                addOutput({ type: 'success', text: `Finished editing ${testName}` });
+            }
+            else {
+                addOutput({ type: 'error', text: `Editor exited with code ${code}` });
+            }
+            resolve(true); // Continue loop
+        });
+        editorProcess.on('error', (err) => {
+            addOutput({ type: 'error', text: `Failed to open editor: ${err.message}` });
+            addOutput({ type: 'info', text: 'Try setting the EDITOR environment variable.' });
+            resolve(true); // Continue loop
+        });
+    });
+}

package/build/src/commands/exit.js

@@ -0,0 +1,20 @@
+/**
+ * Exit command handler
+ */
+/**
+ * Handle /exit command
+ * @param {string} args - Command arguments
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context (includes rl)
+ * @returns {Promise<boolean>} - false to exit loop
+ */
+export async function handleExit(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    addOutput({ type: 'system', text: 'Goodbye!' });
+    // Force immediate exit to stop any ongoing execution
+    // Use setTimeout to allow the "Goodbye!" message to render first
+    setTimeout(() => {
+        process.exit(0);
+    }, 100);
+    return false;
+}

package/build/src/commands/help.js

@@ -0,0 +1,34 @@
+/**
+ * Help command handler
+ */
+/**
+ * Handle /help command
+ * @param {string} args - Command arguments
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleHelp(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    addOutput({ type: 'system', text: 'droid-cua - AI-powered Android testing CLI' });
+    addOutput({ type: 'info', text: '' });
+    addOutput({ type: 'info', text: 'Available commands:' });
+    addOutput({ type: 'info', text: '  /help                    Show this help message' });
+    addOutput({ type: 'info', text: '  /exit                    Exit the CLI' });
+    addOutput({ type: 'info', text: '' });
+    addOutput({ type: 'info', text: 'Test Management:' });
+    addOutput({ type: 'info', text: '  /create <test-name>      Create a new test with autonomous design' });
+    addOutput({ type: 'info', text: '  /run <test-name>         Execute an existing test' });
+    addOutput({ type: 'info', text: '  /list                    List all available tests' });
+    addOutput({ type: 'info', text: '  /view <test-name>        View test contents with line numbers' });
+    addOutput({ type: 'info', text: '  /edit <test-name>        Edit a test in your default editor' });
+    addOutput({ type: 'info', text: '' });
+    addOutput({ type: 'info', text: 'Examples:' });
+    addOutput({ type: 'info', text: '  /create login-test       (design a new test)' });
+    addOutput({ type: 'info', text: '  /list                    (see all tests)' });
+    addOutput({ type: 'info', text: '  /view login-test         (view test contents)' });
+    addOutput({ type: 'info', text: '  /run login-test          (execute test)' });
+    addOutput({ type: 'info', text: '' });
+    addOutput({ type: 'info', text: 'For more info, see README.md' });
+    return true; // Continue loop
+}

package/build/src/commands/index.js

@@ -0,0 +1,49 @@
+/**
+ * Command registry and router
+ */
+import { handleHelp } from './help.js';
+import { handleExit } from './exit.js';
+import { handleRun } from './run.js';
+import { handleCreate } from './create.js';
+import { handleList } from './list.js';
+import { handleView } from './view.js';
+import { handleEdit } from './edit.js';
+import { handleStop } from './stop.js';
+/**
+ * Map of command names to their handlers
+ * Each handler receives (args, session, context)
+ */
+const COMMAND_HANDLERS = {
+    help: handleHelp,
+    exit: handleExit,
+    run: handleRun,
+    create: handleCreate,
+    list: handleList,
+    view: handleView,
+    edit: handleEdit,
+    stop: handleStop,
+};
+/**
+ * Route a command to its handler
+ * @param {string} command - Command name (without /)
+ * @param {string} args - Command arguments
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context (rl, etc.)
+ * @returns {Promise<boolean>} - true if command should continue loop, false to exit
+ */
+export async function routeCommand(command, args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const handler = COMMAND_HANDLERS[command];
+    if (!handler) {
+        addOutput({ type: 'error', text: `Unknown command: /${command}` });
+        addOutput({ type: 'info', text: 'Type /help to see available commands.' });
+        return true; // Continue loop
+    }
+    return await handler(args, session, context);
+}
+/**
+ * Get list of available commands
+ */
+export function getAvailableCommands() {
+    return Object.keys(COMMAND_HANDLERS);
+}

package/build/src/commands/list.js

@@ -0,0 +1,55 @@
+/**
+ * List command handler
+ */
+import { listTests } from "../test-store/test-manager.js";
+/**
+ * Handle /list command
+ * @param {string} args - Command arguments (unused)
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleList(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    addOutput({ type: 'system', text: 'Available tests:' });
+    addOutput({ type: 'info', text: '' });
+    const tests = await listTests();
+    if (tests.length === 0) {
+        addOutput({ type: 'info', text: '  (no tests found)' });
+        addOutput({ type: 'info', text: '' });
+        addOutput({ type: 'info', text: 'Create a test with: /create <test-name>' });
+    }
+    else {
+        for (const test of tests) {
+            // Format date as relative time
+            const now = new Date();
+            const diff = now - test.modified;
+            const minutes = Math.floor(diff / 60000);
+            const hours = Math.floor(minutes / 60);
+            const days = Math.floor(hours / 24);
+            let timeAgo;
+            if (days > 0) {
+                timeAgo = `${days} day${days > 1 ? 's' : ''} ago`;
+            }
+            else if (hours > 0) {
+                timeAgo = `${hours} hour${hours > 1 ? 's' : ''} ago`;
+            }
+            else if (minutes > 0) {
+                timeAgo = `${minutes} minute${minutes > 1 ? 's' : ''} ago`;
+            }
+            else {
+                timeAgo = 'just now';
+            }
+            // Output test name in cyan, metadata in dim
+            addOutput({
+                type: 'test-name',
+                text: `  ${test.name}`,
+                metadata: `(${test.lines} lines, modified ${timeAgo})`
+            });
+        }
+        addOutput({ type: 'info', text: '' });
+        addOutput({ type: 'info', text: 'Run a test with: /run <test-name>' });
+        addOutput({ type: 'info', text: 'View a test with: /view <test-name>' });
+    }
+    return true; // Continue loop
+}

package/build/src/commands/run.js

@@ -0,0 +1,112 @@
+/**
+ * Run command handler
+ */
+import { loadTest, listTests, testExists } from "../test-store/test-manager.js";
+import { ExecutionMode } from "../modes/execution-mode.js";
+import { buildExecutionModePrompt } from "../core/prompts.js";
+/**
+ * Handle /run command
+ * @param {string} args - Test name
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context (includes rl, engine)
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleRun(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const testName = args.trim();
+    // If no test name provided, list available tests
+    if (!testName) {
+        addOutput({ type: 'info', text: 'Usage: /run <test-name>' });
+        addOutput({ type: 'info', text: '' });
+        addOutput({ type: 'info', text: 'Available tests:' });
+        const tests = await listTests();
+        if (tests.length === 0) {
+            addOutput({ type: 'info', text: '  (no tests found)' });
+            addOutput({ type: 'info', text: '' });
+            addOutput({ type: 'info', text: 'Create a test first with: /create <test-name>' });
+        }
+        else {
+            for (const test of tests) {
+                addOutput({ type: 'info', text: `  ${test.name} (${test.lines} lines)` });
+            }
+        }
+        return true; // Continue loop
+    }
+    // Check if test exists
+    const exists = await testExists(testName);
+    if (!exists) {
+        addOutput({ type: 'error', text: `Test not found: ${testName}` });
+        addOutput({ type: 'info', text: 'Use /list to see available tests.' });
+        return true; // Continue loop
+    }
+    // Set mode and test name in Ink UI
+    if (context.setMode) {
+        context.setMode('execution');
+    }
+    if (context.setTestName) {
+        context.setTestName(testName);
+    }
+    // Load test instructions
+    addOutput({ type: 'system', text: `Loading test: ${testName}` });
+    const instructions = await loadTest(testName);
+    addOutput({ type: 'info', text: `Loaded ${instructions.length} instructions` });
+    addOutput({ type: 'info', text: '' });
+    // Disable free-form input during execution (only allow commands like /exit)
+    if (context.setInputDisabled) {
+        context.setInputDisabled(false); // Keep input enabled, but...
+    }
+    if (context.setExecutionMode) {
+        context.setExecutionMode(true); // Signal we're in execution mode
+    }
+    if (context.setInputPlaceholder) {
+        context.setInputPlaceholder('Type /stop or /exit to stop test execution');
+    }
+    // Set agent working status
+    if (context.setAgentWorking) {
+        context.setAgentWorking(true, 'Executing test...');
+    }
+    // Reset session state to prevent context leak from design mode
+    // Each test instruction should execute in isolation
+    session.updateResponseId(undefined);
+    session.clearMessages();
+    // Set execution mode system prompt (replaces any design mode prompt)
+    const executionPrompt = buildExecutionModePrompt(session.deviceInfo);
+    session.setSystemPrompt(executionPrompt);
+    // Create execution mode
+    const executionMode = new ExecutionMode(session, context.engine, instructions);
+    // Store reference in context so /stop command can access it
+    if (context.setActiveExecutionMode) {
+        context.setActiveExecutionMode(executionMode);
+    }
+    // Execute the test
+    const result = await executionMode.execute(context);
+    // Clear agent working status
+    if (context.setAgentWorking) {
+        context.setAgentWorking(false);
+    }
+    // Clear execution mode reference
+    if (context.setActiveExecutionMode) {
+        context.setActiveExecutionMode(null);
+    }
+    // Re-enable free-form input
+    if (context.setExecutionMode) {
+        context.setExecutionMode(false);
+    }
+    if (context.setInputPlaceholder) {
+        context.setInputPlaceholder('Type a command or message...');
+    }
+    // Reset mode
+    if (context.setMode) {
+        context.setMode('command');
+    }
+    if (context.setTestName) {
+        context.setTestName(null);
+    }
+    if (result.success) {
+        addOutput({ type: 'success', text: '✓ Test passed!' });
+    }
+    else {
+        addOutput({ type: 'error', text: `✗ Test failed: ${result.error || "Unknown error"}` });
+    }
+    return true; // Continue loop
+}

package/build/src/commands/stop.js

@@ -0,0 +1,32 @@
+/**
+ * Stop command handler - stops current test creation or execution
+ */
+/**
+ * Handle /stop command
+ * @param {string} args - Command arguments (unused)
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleStop(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    // Check if we're in design mode
+    if (context.activeDesignMode) {
+        addOutput({ type: 'info', text: 'Stopping test creation...' });
+        // Signal design mode to stop by queuing "cancel"
+        context.activeDesignMode.handleUserInput('cancel');
+        return true; // Continue loop
+    }
+    // Check if we're in execution mode
+    if (context.isExecutionMode) {
+        addOutput({ type: 'info', text: 'Stopping test execution...' });
+        // Set flag to stop execution
+        if (context.activeExecutionMode) {
+            context.activeExecutionMode.shouldStop = true;
+        }
+        return true; // Continue loop
+    }
+    // Not in any mode that can be stopped
+    addOutput({ type: 'info', text: 'No active test creation or execution to stop.' });
+    return true; // Continue loop
+}

package/build/src/commands/view.js

@@ -0,0 +1,43 @@
+/**
+ * View command handler
+ */
+import { testExists, getTestContent } from "../test-store/test-manager.js";
+/**
+ * Handle /view command
+ * @param {string} args - Test name
+ * @param {Object} session - Current session
+ * @param {Object} context - Additional context
+ * @returns {Promise<boolean>} - true to continue loop
+ */
+export async function handleView(args, session, context) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const testName = args.trim();
+    // Check if test name provided
+    if (!testName) {
+        addOutput({ type: 'error', text: 'Usage: /view <test-name>' });
+        addOutput({ type: 'info', text: '' });
+        addOutput({ type: 'info', text: 'Example:' });
+        addOutput({ type: 'info', text: '  /view example' });
+        return true; // Continue loop
+    }
+    // Check if test exists
+    const exists = await testExists(testName);
+    if (!exists) {
+        addOutput({ type: 'error', text: `Test not found: ${testName}` });
+        addOutput({ type: 'info', text: 'Use /list to see available tests.' });
+        return true; // Continue loop
+    }
+    // Load and display test content
+    const content = await getTestContent(testName);
+    const lines = content.split('\n');
+    addOutput({ type: 'system', text: `Test: ${testName}` });
+    addOutput({ type: 'info', text: '─'.repeat(60) });
+    // Display with line numbers
+    lines.forEach((line, index) => {
+        const lineNum = String(index + 1).padStart(3, ' ');
+        addOutput({ type: 'info', text: `${lineNum} │ ${line}` });
+    });
+    addOutput({ type: 'info', text: '─'.repeat(60) });
+    addOutput({ type: 'info', text: `${lines.length} lines total` });
+    return true; // Continue loop
+}

package/build/src/core/execution-engine.js

@@ -0,0 +1,114 @@
+import path from "path";
+import { writeFile } from "fs/promises";
+import { getScreenshotAsBase64 } from "../device/connection.js";
+import { handleModelAction } from "../device/actions.js";
+import { sendCUARequest } from "../device/openai.js";
+export class ExecutionEngine {
+    constructor(session, options = {}) {
+        this.session = session;
+        this.recordScreenshots = options.recordScreenshots || false;
+        this.screenshotDir = options.screenshotDir || null;
+    }
+    /**
+     * Run a full turn with the CUA model
+     * Executes actions until the model stops requesting actions
+     * Returns the new response ID for chaining
+     * @param {Object} response - The CUA response
+     * @param {Function} trackAction - Optional callback to track actions for stuck detection
+     * @param {Object} context - Optional Ink context for output
+     */
+    async runFullTurn(response, trackAction = null, context = null) {
+        const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+        let newResponseId = response.id;
+        while (true) {
+            // Check for interruption before processing next batch of actions
+            if (trackAction) {
+                const shouldStop = trackAction(null); // null action = pre-batch check
+                if (shouldStop) {
+                    return newResponseId;
+                }
+            }
+            const items = response.output || [];
+            const actions = items.filter(item => item.type === "computer_call");
+            // ── Collect pending safety checks ──
+            const pendingSafetyChecks = items
+                .filter(item => item.type === "pending_safety_check")
+                .map(item => ({ id: item.id }));
+            // ── Print assistant output ──
+            for (const item of items) {
+                if (item.type === "reasoning") {
+                    for (const entry of item.summary) {
+                        if (entry.type === "summary_text") {
+                            addOutput({ type: 'reasoning', text: entry.text });
+                            this.session.addToTranscript(`[Reasoning] ${entry.text}`);
+                        }
+                    }
+                }
+                else if (item.type === "message") {
+                    const textPart = item.content.find(c => c.type === "output_text");
+                    if (textPart) {
+                        addOutput({ type: 'assistant', text: textPart.text });
+                        this.session.addToTranscript(`[Assistant] ${textPart.text}`);
+                    }
+                }
+                else if (item.type === "pending_safety_check") {
+                    addOutput({ type: 'warning', text: `⚠️ Safety check: ${item.code} - ${item.message}` });
+                }
+            }
+            if (actions.length === 0) {
+                // No actions = turn complete
+                break;
+            }
+            // ── Process model actions ──
+            for (const { action, call_id } of actions) {
+                if (action.type === "screenshot") {
+                    addOutput({ type: 'info', text: '📸 Capturing screen' });
+                }
+                else {
+                    await handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, context);
+                    // Track action and check for interruption
+                    if (trackAction) {
+                        const shouldStop = trackAction(action);
+                        if (shouldStop) {
+                            // User interrupted - stop execution immediately
+                            return newResponseId;
+                        }
+                    }
+                    // Add delay after UI-changing actions to let the interface update
+                    // before taking the screenshot (except for explicit wait actions which have their own delay)
+                    if (action.type !== "wait") {
+                        await new Promise(resolve => setTimeout(resolve, 500));
+                    }
+                }
+                const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+                if (this.recordScreenshots && this.screenshotDir) {
+                    const framePath = path.join(this.screenshotDir, `frame_${String(Date.now())}.png`);
+                    await writeFile(framePath, Buffer.from(screenshotBase64, "base64"));
+                }
+                // Build next input: screenshot + any carryover reasoning
+                const input = [{
+                        type: "computer_call_output",
+                        call_id,
+                        output: {
+                            type: "computer_screenshot",
+                            image_url: `data:image/png;base64,${screenshotBase64}`,
+                        },
+                        current_url: "android://emulator", // Android emulator doesn't have URLs like a browser
+                        ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
+                    }];
+                response = await sendCUARequest({
+                    messages: input,
+                    previousResponseId: newResponseId,
+                    deviceInfo: this.session.deviceInfo,
+                });
+                newResponseId = response.id;
+            }
+        }
+        // ── At end, if last output was only reasoning ──
+        const finalItems = response.output || [];
+        if (finalItems.length > 0 && finalItems.at(-1).type === "reasoning") {
+            addOutput({ type: 'info', text: 'Warning: last item was reasoning without follow-up. Dropping to avoid 400 error.' });
+        }
+        return newResponseId;
+    }
+}