@loadmill/droid-cua 1.1.1 → 2.0.0

package/build/src/modes/execution-mode.js

@@ -3,6 +3,7 @@ import { sendCUARequest } from "../device/openai.js";
 import { isAssertion, extractAssertionPrompt, buildAssertionSystemPrompt, checkAssertionResult, handleAssertionFailure, handleAssertionSuccess, } from "../device/assertions.js";
 import { isLoadmillInstruction, extractLoadmillCommand, executeLoadmillInstruction, } from "../device/loadmill.js";
 import { logger } from "../utils/logger.js";
+import { emitDesktopDebug } from "../utils/desktop-debug.js";
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -15,6 +16,63 @@ export class ExecutionMode {
         this.initialSystemText = session.systemPrompt;
         this.shouldStop = false; // Flag to stop execution (set by /stop command)
         this.isHeadlessMode = isHeadlessMode; // true for CI/automated runs, false for interactive
+        // Stats tracking
+        this.stats = {
+            startTime: null,
+            actionCount: 0,
+            instructionsCompleted: 0,
+            retryCount: 0,
+            assertionsPassed: 0,
+            assertionsFailed: 0,
+        };
+    }
+    /**
+     * Format duration in human-readable format (Xm Ys)
+     */
+    formatDuration(ms) {
+        const totalSeconds = Math.floor(ms / 1000);
+        const minutes = Math.floor(totalSeconds / 60);
+        const seconds = totalSeconds % 60;
+        if (minutes > 0) {
+            return `${minutes}m ${seconds}s`;
+        }
+        return `${seconds}s`;
+    }
+    /**
+     * Format stats for display
+     */
+    formatStats() {
+        const duration = Date.now() - this.stats.startTime;
+        const instructionCount = this.instructions.length;
+        const totalAssertions = this.stats.assertionsPassed + this.stats.assertionsFailed;
+        const lines = [
+            '',
+            `  Duration:     ${this.formatDuration(duration)}`,
+            `  Steps:        ${this.stats.actionCount} actions (${instructionCount} instructions)`,
+        ];
+        if (totalAssertions > 0) {
+            lines.push(`  Assertions:   ${this.stats.assertionsPassed}/${totalAssertions} passed`);
+        }
+        lines.push(`  Retries:      ${this.stats.retryCount}`);
+        return lines;
+    }
+    buildStepContext(instructionIndex) {
+        return {
+            instructionIndex,
+            stepId: `step-${String(instructionIndex + 1).padStart(4, "0")}`,
+        };
+    }
+    emit(addOutput, type, text, context = {}, stepContext = null, extra = {}) {
+        addOutput({
+            type,
+            text,
+            eventType: extra.eventType,
+            actionType: extra.actionType,
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+            payload: extra.payload
+        });
     }
     /**
      * Execute all instructions in the test script
@@ -22,25 +80,44 @@ export class ExecutionMode {
      * @returns {Promise<{success: boolean, error?: string}>}
      */
     async execute(context = {}) {
-        const addOutput = context.addOutput || ((item) => console.log(item.text || item));
+        const runContext = {
+            ...context,
+            runId: context.runId || `run-${Date.now()}`
+        };
+        const addOutput = runContext.addOutput || ((item) => console.log(item.text || item));
+        // Start timing
+        this.stats.startTime = Date.now();
         for (let i = 0; i < this.instructions.length; i++) {
+            const stepContext = this.buildStepContext(i);
             // Check if execution should be stopped
             if (this.shouldStop) {
-                addOutput({ type: 'info', text: 'Test execution stopped by user.' });
+                this.emit(addOutput, 'info', 'Test execution stopped by user.', runContext, stepContext, {
+                    eventType: 'system_message'
+                });
                 return { success: false, error: 'Stopped by user' };
             }
             const instruction = this.instructions[i];
-            addOutput({ type: 'user', text: instruction });
+            this.emit(addOutput, 'user', instruction, runContext, stepContext, {
+                eventType: 'instruction_started',
+                payload: {
+                    instruction,
+                    isAssertion: isAssertion(instruction)
+                }
+            });
             // Check for exit command
             if (instruction.toLowerCase() === "exit") {
-                addOutput({ type: 'success', text: 'Test completed.' });
+                this.stats.instructionsCompleted++;
+                this.emit(addOutput, 'success', 'Test completed.', runContext, stepContext, {
+                    eventType: 'system_message'
+                });
                 return { success: true };
             }
             try {
-                const result = await this.executeInstruction(instruction, context);
+                const result = await this.executeInstruction(instruction, runContext, 0, stepContext);
                 if (!result.success) {
                     return result; // Propagate failure
                 }
+                this.stats.instructionsCompleted++;
             }
             catch (err) {
                 // Log full error details to file
@@ -54,13 +131,36 @@ export class ExecutionMode {
                     stack: err.stack
                 });
                 // Show user-friendly error message
-                addOutput({ type: 'error', text: `Error executing instruction: ${instruction}` });
-                addOutput({ type: 'error', text: err.message });
-                addOutput({ type: 'info', text: 'Full error details have been logged to the debug log.' });
+                this.emit(addOutput, 'error', `Error executing instruction: ${instruction}`, runContext, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        instruction,
+                        message: err.message,
+                        status: err.status,
+                        code: err.code
+                    }
+                });
+                this.emit(addOutput, 'error', err.message, runContext, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: err.message,
+                        status: err.status,
+                        code: err.code
+                    }
+                });
+                this.emit(addOutput, 'info', 'Full error details have been logged to the debug log.', runContext, stepContext, {
+                    eventType: 'system_message'
+                });
                 return { success: false, error: err.message };
             }
         }
-        addOutput({ type: 'success', text: 'Test completed successfully.' });
+        this.emit(addOutput, 'success', 'Test completed successfully.', runContext, null, {
+            eventType: 'system_message'
+        });
+        // Display stats
+        for (const line of this.formatStats()) {
+            addOutput({ type: 'info', text: line });
+        }
         return { success: true };
     }
     /**
@@ -70,23 +170,25 @@ export class ExecutionMode {
      * @param {number} retryCount - Current retry attempt (internal use)
      * @returns {Promise<{success: boolean, error?: string}>}
      */
-    async executeInstruction(instruction, context, retryCount = 0) {
-        const MAX_RETRIES = 3;
+    async executeInstruction(instruction, context, retryCount = 0, stepContext = null) {
+        const MAX_RETRIES = 10;
         const addOutput = context.addOutput || ((item) => console.log(item.text || item));
         // ── Check for Loadmill instruction ──
         if (isLoadmillInstruction(instruction)) {
             const loadmillCommand = extractLoadmillCommand(instruction);
             this.session.addToTranscript(`[Loadmill] ${loadmillCommand}`);
-            const result = await executeLoadmillInstruction(loadmillCommand, this.isHeadlessMode, context);
+            const result = await executeLoadmillInstruction(loadmillCommand, this.isHeadlessMode, context, stepContext);
             // Handle retry request from interactive mode
             if (result.retry) {
-                return await this.executeInstruction(instruction, context);
+                this.stats.retryCount++;
+                return await this.executeInstruction(instruction, context, 0, stepContext);
             }
             return result;
         }
         // ── Check for assertion ──
         const isAssertionStep = isAssertion(instruction);
         let assertionPrompt = null;
+        const transcriptStartIndex = this.session.transcript.length;
         if (isAssertionStep) {
             assertionPrompt = extractAssertionPrompt(instruction);
             const assertionSystemPrompt = buildAssertionSystemPrompt(this.initialSystemText, assertionPrompt);
@@ -104,6 +206,7 @@ export class ExecutionMode {
             // When continuing with previousResponseId, only send the new instruction
             // The server already has full context from previous responses
             let messagesToSend;
+            const previousResponseIdToSend = isAssertionStep ? null : this.session.previousResponseId;
             if (this.session.previousResponseId && !isAssertionStep) {
                 // Only send the new user instruction
                 messagesToSend = [{ role: "user", content: instruction }];
@@ -115,23 +218,43 @@ export class ExecutionMode {
             const response = await sendCUARequest({
                 messages: messagesToSend,
                 screenshotBase64,
-                previousResponseId: this.session.previousResponseId,
+                previousResponseId: previousResponseIdToSend,
                 deviceInfo: this.session.deviceInfo,
+                debugContext: {
+                    scope: "execution",
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }
             });
-            const newResponseId = await this.engine.runFullTurn(response, null, context);
+            // Track actions for stats
+            const trackAction = (action) => {
+                if (action && action.type !== 'screenshot') {
+                    this.stats.actionCount++;
+                }
+                return false; // Don't stop execution
+            };
+            const newResponseId = await this.engine.runFullTurn(response, trackAction, context, stepContext);
             this.session.updateResponseId(newResponseId);
             // ── Check assertion result ──
             if (isAssertionStep) {
-                const result = checkAssertionResult(this.session.transcript);
+                const assertionTranscript = this.session.transcript.slice(transcriptStartIndex);
+                const result = checkAssertionResult(assertionTranscript);
                 if (result.failed) {
-                    handleAssertionFailure(assertionPrompt, this.session.transcript, false, // Never exit process - we'll always prompt the user in interactive mode
-                    context);
+                    handleAssertionFailure(assertionPrompt, assertionTranscript, false, // Never exit process - we'll always prompt the user in interactive mode
+                    context, stepContext);
                     // In headless mode, exit immediately on assertion failure
                     if (this.isHeadlessMode) {
+                        this.stats.assertionsFailed++;
                         return { success: false, error: `Assertion failed: ${assertionPrompt}` };
                     }
                     // Interactive mode - ask user what to do
-                    addOutput({ type: 'system', text: 'What would you like to do? (retry/skip/stop)' });
+                    this.emit(addOutput, 'system', 'What would you like to do? (retry/skip/stop)', context, stepContext, {
+                        eventType: 'input_request',
+                        payload: {
+                            options: ['retry', 'skip', 'stop']
+                        }
+                    });
                     // Wait for user input
                     const userChoice = await new Promise((resolve) => {
                         if (context?.waitForUserInput) {
@@ -145,19 +268,25 @@ export class ExecutionMode {
                     const choice = userChoice.toLowerCase().trim();
                     if (choice === 'retry' || choice === 'r') {
                         // Retry the same instruction by recursing
-                        return await this.executeInstruction(instruction, context);
+                        this.stats.retryCount++;
+                        return await this.executeInstruction(instruction, context, 0, stepContext);
                     }
                     else if (choice === 'skip' || choice === 's') {
                         // Continue to next instruction
-                        addOutput({ type: 'info', text: 'Skipping failed assertion and continuing...' });
+                        this.stats.assertionsFailed++;
+                        this.emit(addOutput, 'info', 'Skipping failed assertion and continuing...', context, stepContext, {
+                            eventType: 'system_message'
+                        });
                     }
                     else {
                         // Stop execution
+                        this.stats.assertionsFailed++;
                         return { success: false, error: `Assertion failed: ${assertionPrompt}` };
                     }
                 }
                 else if (result.passed) {
-                    handleAssertionSuccess(assertionPrompt, context);
+                    this.stats.assertionsPassed++;
+                    handleAssertionSuccess(assertionPrompt, context, stepContext);
                 }
             }
             // Clear messages after each turn but KEEP the response chain for context
@@ -181,9 +310,28 @@ export class ExecutionMode {
             const addOutput = context.addOutput || ((item) => console.log(item.text || item));
             // Check if we've exceeded max retries
             if (retryCount >= MAX_RETRIES) {
-                addOutput({ type: 'error', text: `Failed after ${MAX_RETRIES} retries. Device may be disconnected.` });
+                emitDesktopDebug("reconnect.attempt", "device", {
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }, {
+                    stage: "start",
+                    reason: err.message,
+                    attemptsExhausted: retryCount,
+                    maxRetries: MAX_RETRIES
+                });
+                this.emit(addOutput, 'error', `Failed after ${MAX_RETRIES} retries. Device may be disconnected.`, context, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: `Failed after ${MAX_RETRIES} retries. Device may be disconnected.`,
+                        attempt: retryCount,
+                        maxRetries: MAX_RETRIES
+                    }
+                });
                 // Attempt to reconnect to the device
-                addOutput({ type: 'info', text: 'Attempting to reconnect to device...' });
+                this.emit(addOutput, 'info', 'Attempting to reconnect to device...', context, stepContext, {
+                    eventType: 'system_message'
+                });
                 try {
                     const platform = getCurrentPlatform();
                     const deviceName = this.session.deviceName || undefined;
@@ -192,17 +340,65 @@ export class ExecutionMode {
                     // Update session with new connection
                     this.session.deviceId = deviceId;
                     this.session.deviceInfo = deviceInfo;
-                    addOutput({ type: 'success', text: 'Reconnected to device. Resuming...' });
+                    emitDesktopDebug("reconnect.attempt", "device", {
+                        runId: context?.runId,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex
+                    }, {
+                        stage: "success",
+                        deviceId
+                    });
+                    this.emit(addOutput, 'success', 'Reconnected to device. Resuming...', context, stepContext, {
+                        eventType: 'system_message'
+                    });
                     // Reset retry count and try again
-                    return await this.executeInstruction(instruction, context, 0);
+                    return await this.executeInstruction(instruction, context, 0, stepContext);
                 }
                 catch (reconnectErr) {
+                    emitDesktopDebug("reconnect.attempt", "device", {
+                        runId: context?.runId,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex
+                    }, {
+                        stage: "failed",
+                        message: reconnectErr.message
+                    });
                     logger.error('Failed to reconnect to device', { error: reconnectErr.message });
-                    addOutput({ type: 'error', text: `Could not reconnect to device: ${reconnectErr.message}` });
+                    this.emit(addOutput, 'error', `Could not reconnect to device: ${reconnectErr.message}`, context, stepContext, {
+                        eventType: 'error',
+                        payload: {
+                            message: reconnectErr.message
+                        }
+                    });
                     return { success: false, error: 'Device disconnected and reconnection failed' };
                 }
             }
-            addOutput({ type: 'info', text: `Connection issue. Retrying... (${retryCount + 1}/${MAX_RETRIES})` });
+            this.emit(addOutput, 'info', 'Thinking...', context, stepContext, {
+                eventType: 'system_message',
+                payload: {
+                    attempt: retryCount + 1,
+                    maxRetries: MAX_RETRIES,
+                    reason: err.message
+                }
+            });
+            emitDesktopDebug("retry.attempt", "device", {
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex
+            }, {
+                attempt: retryCount + 1,
+                maxRetries: MAX_RETRIES,
+                reason: err.message
+            });
+            emitDesktopDebug("device.disconnect", "device", {
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex
+            }, {
+                reason: err.message
+            });
+            // Track retry for stats
+            this.stats.retryCount++;
             // Build context for retry - include transcript in system message to avoid conversational responses
             const transcriptContext = this.session.getTranscriptText();
             this.session.clearMessages();
@@ -218,7 +414,7 @@ export class ExecutionMode {
             // Wait a bit before retrying
             await new Promise(resolve => setTimeout(resolve, 1000));
             // Retry the same instruction with incremented counter
-            return await this.executeInstruction(instruction, context, retryCount + 1);
+            return await this.executeInstruction(instruction, context, retryCount + 1, stepContext);
         }
     }
 }

package/build/src/test-store/test-manager.js

@@ -1,8 +1,7 @@
-import { readdir, readFile, writeFile, unlink, stat } from "fs/promises";
+import { readdir, readFile, writeFile, unlink, stat, mkdir } from "fs/promises";
 import path from "path";
-import { fileURLToPath } from "url";
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const TESTS_DIR = path.join(__dirname, "../../tests");
+// Tests directory is relative to current working directory
+const TESTS_DIR = path.join(process.cwd(), "tests");
 /**
  * Save a test script to the tests/ directory
  * @param {string} name - Test name (without .dcua extension)
@@ -14,6 +13,8 @@ export async function saveTest(name, content) {
     const cleanName = name.endsWith(".dcua") ? name.slice(0, -5) : name;
     const filename = `${cleanName}.dcua`;
     const filepath = path.join(TESTS_DIR, filename);
+    // Create tests directory if it doesn't exist
+    await mkdir(TESTS_DIR, { recursive: true });
     await writeFile(filepath, content, "utf-8");
     return filepath;
 }
@@ -46,6 +47,13 @@ export async function getTestContent(name) {
  * @returns {Promise<Array<{name: string, path: string, lines: number, modified: Date}>>}
  */
 export async function listTests() {
+    // Return empty array if tests directory doesn't exist
+    try {
+        await stat(TESTS_DIR);
+    }
+    catch {
+        return [];
+    }
     const files = await readdir(TESTS_DIR);
     const dcuaFiles = files.filter(f => f.endsWith(".dcua"));
     const tests = await Promise.all(dcuaFiles.map(async (filename) => {