@loadmill/droid-cua 1.1.2 → 2.0.0

package/build/src/device/ios/actions.js

@@ -6,6 +6,29 @@
 import * as appium from "./appium-client.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { logger } from "../../utils/logger.js";
+import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
+function normalizeMobileKeypress(keys = []) {
+    if (!Array.isArray(keys) || keys.length === 0) {
+        throw new Error("Keypress action is missing keys");
+    }
+    if (keys.length > 1) {
+        throw new Error(`Unsupported mobile key chord: ${keys.join(", ")}. Use taps and text entry instead.`);
+    }
+    const key = String(keys[0]).trim().toUpperCase();
+    if (key === "ESC" || key === "ESCAPE" || key === "HOME") {
+        return { kind: "button", originalKey: keys[0], mapped: "home" };
+    }
+    if (key === "ENTER" || key === "RETURN") {
+        return { kind: "text", originalKey: keys[0], mapped: "\n", label: "Return key" };
+    }
+    if (key === "BACKSPACE" || key === "DELETE") {
+        return { kind: "text", originalKey: keys[0], mapped: "\b", label: "Delete key" };
+    }
+    if (key === "SPACE") {
+        return { kind: "text", originalKey: keys[0], mapped: " ", label: "Space key" };
+    }
+    throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
+}
 /**
  * Handle an action from the CUA model
  * @param {string} simulatorId - The simulator UDID
@@ -16,24 +39,50 @@ import { logger } from "../../utils/logger.js";
 export async function handleModelAction(simulatorId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const session = getActiveSession();
+    const meta = (payload = {}) => ({
+        eventType: "tool_call",
+        actionType: action?.type,
+        runId: context?.runId,
+        stepId: context?.stepId,
+        instructionIndex: context?.instructionIndex,
+        payload: {
+            platform: "ios",
+            ...payload
+        }
+    });
     if (!session) {
         throw new Error("No active iOS session");
     }
     try {
+        emitDesktopDebug("device.action.execute", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex
+        }, {
+            platform: "ios",
+            simulatorId,
+            actionType: action?.type,
+            scale,
+            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
+            pathPoints: Array.isArray(action?.path) ? action.path.length : 0
+        });
         switch (action.type) {
-            case "click": {
+            case "click":
+            case "double_click": {
                 // Convert scaled coordinates to pixels, then to logical points for Appium
                 const dpr = getDevicePixelRatio();
                 const pixelX = Math.round(action.x / scale);
                 const pixelY = Math.round(action.y / scale);
                 const pointX = Math.round(pixelX / dpr);
                 const pointY = Math.round(pixelY / dpr);
-                addOutput({ type: "action", text: `Tapping at (${pointX}, ${pointY}) points` });
+                addOutput({ type: "action", text: `Tapping at (${pointX}, ${pointY}) points`, ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
                 await appium.tap(session.sessionId, pointX, pointY);
                 break;
             }
             case "type": {
-                addOutput({ type: "action", text: `Typing text: ${action.text}` });
+                addOutput({ type: "action", text: `Typing text: ${action.text}`, ...meta({ text: action.text }) });
                 await appium.type(session.sessionId, action.text);
                 break;
             }
@@ -41,7 +90,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 const dpr = getDevicePixelRatio();
                 const scrollX = Math.round((action.scroll_x / scale) / dpr);
                 const scrollY = Math.round((action.scroll_y / scale) / dpr);
-                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY}) points` });
+                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY}) points`, ...meta({ scrollX, scrollY, unit: "points" }) });
                 // Start from center of screen (in logical points)
                 const centerX = 197; // Center of iPhone 16 (393/2)
                 const centerY = 426; // Center of iPhone 16 (852/2)
@@ -64,6 +113,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                     addOutput({
                         type: "action",
                         text: `Dragging from (${startX}, ${startY}) to (${endX}, ${endY}) points`,
+                        ...meta({ pathStart: { x: startX, y: startY }, pathEnd: { x: endX, y: endY }, unit: "points" })
                     });
                     await appium.drag(session.sessionId, startX, startY, endX, endY);
                 }
@@ -73,31 +123,19 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 break;
             }
             case "keypress": {
-                const { keys } = action;
-                for (const key of keys) {
-                    const upperKey = key.toUpperCase();
-                    if (upperKey === "ESC" || upperKey === "ESCAPE") {
-                        // Map ESC to home button on iOS
-                        addOutput({ type: "action", text: "Pressing Home button" });
-                        await appium.pressButton(session.sessionId, "home");
-                    }
-                    else if (upperKey === "ENTER" || upperKey === "RETURN") {
-                        addOutput({ type: "action", text: "Pressing Return key" });
-                        await appium.type(session.sessionId, "\n");
-                    }
-                    else if (upperKey === "BACKSPACE" || upperKey === "DELETE") {
-                        addOutput({ type: "action", text: "Pressing Delete key" });
-                        await appium.type(session.sessionId, "\b");
-                    }
-                    else {
-                        addOutput({ type: "action", text: `Pressing key: ${key}` });
-                        await appium.type(session.sessionId, key);
-                    }
+                const normalized = normalizeMobileKeypress(action.keys);
+                if (normalized.kind === "button") {
+                    addOutput({ type: "action", text: "Pressing Home button", ...meta({ keys: [normalized.originalKey], mapped: normalized.mapped }) });
+                    await appium.pressButton(session.sessionId, normalized.mapped);
+                }
+                else {
+                    addOutput({ type: "action", text: `Pressing ${normalized.label}`, ...meta({ keys: [normalized.originalKey], mapped: normalized.label }) });
+                    await appium.type(session.sessionId, normalized.mapped);
                 }
                 break;
             }
             case "wait": {
-                addOutput({ type: "action", text: "Waiting..." });
+                addOutput({ type: "action", text: "Waiting...", ...meta({}) });
                 await new Promise((resolve) => setTimeout(resolve, 1000));
                 break;
             }
@@ -111,7 +149,30 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             message: error.message,
             stack: error.stack,
         });
-        addOutput({ type: "error", text: `Error executing action: ${error.message}` });
+        emitDesktopDebug("device.error", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex
+        }, {
+            platform: "ios",
+            operation: "action.execute",
+            actionType: action?.type,
+            message: error.message
+        });
+        addOutput({
+            type: "error",
+            text: `Error executing action: ${error.message}`,
+            eventType: "error",
+            actionType: action?.type,
+            runId: context?.runId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex,
+            payload: {
+                message: error.message,
+                platform: "ios"
+            }
+        });
         addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
     }
 }

package/build/src/device/ios/appium-server.js

@@ -5,16 +5,64 @@
  * Auto-starts if not running, cleans up on process exit.
  */
 import { spawn } from "child_process";
-const APPIUM_URL = process.env.APPIUM_URL || "http://localhost:4723";
+import { existsSync, readdirSync } from "fs";
+import path from "path";
 let appiumProcess = null;
 let cleanupRegistered = false;
+function getAppiumBaseUrl() {
+    return process.env.APPIUM_URL || "http://localhost:4723";
+}
+function collectNvmAppiumCandidates() {
+    const home = process.env.HOME;
+    if (!home)
+        return [];
+    const nvmNodeVersionsDir = path.join(home, ".nvm", "versions", "node");
+    if (!existsSync(nvmNodeVersionsDir))
+        return [];
+    try {
+        return readdirSync(nvmNodeVersionsDir)
+            .map((versionDir) => path.join(nvmNodeVersionsDir, versionDir, "bin", "appium"))
+            .filter((candidate) => existsSync(candidate))
+            .reverse();
+    }
+    catch {
+        return [];
+    }
+}
+export function resolveAppiumCommand() {
+    const explicit = process.env.DROID_CUA_APPIUM_PATH?.trim() || process.env.APPIUM_PATH?.trim();
+    if (explicit) {
+        return explicit;
+    }
+    const commonCandidates = [
+        "/opt/homebrew/bin/appium",
+        "/usr/local/bin/appium",
+        ...collectNvmAppiumCandidates()
+    ];
+    for (const candidate of commonCandidates) {
+        if (existsSync(candidate)) {
+            return candidate;
+        }
+    }
+    return "appium";
+}
+async function spawnAppiumProcess(command) {
+    return await new Promise((resolve, reject) => {
+        const child = spawn(command, [], {
+            detached: true,
+            stdio: "ignore"
+        });
+        child.once("error", reject);
+        child.once("spawn", () => resolve(child));
+    });
+}
 /**
  * Check if Appium server is running
  * @returns {Promise<boolean>}
  */
 export async function isAppiumRunning() {
     try {
-        const response = await fetch(`${APPIUM_URL}/status`);
+        const response = await fetch(`${getAppiumBaseUrl()}/status`);
         return response.ok;
     }
     catch {
@@ -45,11 +93,17 @@ export async function startAppium() {
         console.log("Appium server is already running");
         return;
     }
-    console.log("Starting Appium server...");
-    appiumProcess = spawn("appium", [], {
-        detached: true,
-        stdio: "ignore"
-    });
+    const appiumCommand = resolveAppiumCommand();
+    console.log(`Starting Appium server (${appiumCommand})...`);
+    try {
+        appiumProcess = await spawnAppiumProcess(appiumCommand);
+    }
+    catch (error) {
+        if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
+            throw new Error('Appium CLI not found. Install Appium or set an "iOS Appium Path" in Settings. Finder-launched apps may not inherit your shell PATH.');
+        }
+        throw error;
+    }
     appiumProcess.unref();
     const ready = await waitForAppiumReady(30000);
     if (!ready) {
@@ -97,5 +151,5 @@ export function setupAppiumCleanup() {
  * @returns {string}
  */
 export function getAppiumUrl() {
-    return APPIUM_URL;
+    return getAppiumBaseUrl();
 }

package/build/src/device/ios/connection.js

@@ -9,6 +9,7 @@ import sharp from "sharp";
 import { startAppium, setupAppiumCleanup } from "./appium-server.js";
 import * as appium from "./appium-client.js";
 import { logger } from "../../utils/logger.js";
+import { emitDesktopDebug } from "../../utils/desktop-debug.js";
 const execAsync = promisify(exec);
 // Active session state
 let activeSession = null;
@@ -142,6 +143,7 @@ async function waitForSimulatorBoot(udid, timeoutMs = 60000) {
  * @returns {Promise<string>} Simulator ID for use with other functions
  */
 export async function connectToDevice(simulatorName) {
+    emitDesktopDebug("device.connect", "device", {}, { platform: "ios", stage: "start", simulatorName: simulatorName || null });
     // Setup cleanup handlers
     setupAppiumCleanup();
     // Start Appium if not running
@@ -175,6 +177,12 @@ export async function connectToDevice(simulatorName) {
         simulatorName,
     };
     console.log(`Connected to simulator "${simulatorName}" (${udid})`);
+    emitDesktopDebug("device.connect", "device", {}, {
+        platform: "ios",
+        stage: "success",
+        simulatorName,
+        udid
+    });
     return udid;
 }
 /**
@@ -204,6 +212,9 @@ export async function getDeviceInfo(simulatorId) {
     const scaledWidth = Math.round(pixelWidth * scale);
     const scaledHeight = Math.round(pixelHeight * scale);
     return {
+        platform: "ios",
+        device_name: activeSession?.simulatorName || simulatorId,
+        model: activeSession?.simulatorName || simulatorId,
         device_width: pixelWidth,
         device_height: pixelHeight,
         scaled_width: scaledWidth,
@@ -219,8 +230,8 @@ export async function getDeviceInfo(simulatorId) {
  */
 export async function getScreenshotAsBase64(simulatorId, deviceInfo) {
     await ensureSessionAlive();
-    const base64 = await appium.getScreenshot(activeSession.sessionId);
-    let buffer = Buffer.from(base64, "base64");
+    const rawBase64 = await appium.getScreenshot(activeSession.sessionId);
+    let buffer = Buffer.from(rawBase64, "base64");
     logger.debug(`iOS screenshot captured: ${buffer.length} bytes before scaling`);
     if (deviceInfo.scale < 1.0) {
         buffer = await sharp(buffer)
@@ -229,7 +240,15 @@ export async function getScreenshotAsBase64(simulatorId, deviceInfo) {
             .toBuffer();
         logger.debug(`iOS screenshot scaled: ${buffer.length} bytes after scaling`);
     }
-    return buffer.toString("base64");
+    const base64 = buffer.toString("base64");
+    emitDesktopDebug("device.screenshot", "device", {}, {
+        platform: "ios",
+        simulatorId,
+        width: deviceInfo?.scaled_width,
+        height: deviceInfo?.scaled_height,
+        base64Length: base64.length
+    });
+    return base64;
 }
 /**
  * Ensure the Appium session is still alive, recreate if dead
@@ -240,6 +259,15 @@ async function ensureSessionAlive() {
     }
     const status = await appium.getSessionStatus(activeSession.sessionId);
     if (!status) {
+        emitDesktopDebug("reconnect.attempt", "device", {}, {
+            platform: "ios",
+            stage: "start",
+            reason: "appium session status check failed"
+        });
+        emitDesktopDebug("device.disconnect", "device", {}, {
+            platform: "ios",
+            reason: "appium session no longer active"
+        });
         console.log("Session died, recreating...");
         const session = await appium.createSession({
             platformName: "iOS",
@@ -250,6 +278,11 @@ async function ensureSessionAlive() {
             "appium:shouldTerminateApp": false,
         });
         activeSession.sessionId = session.sessionId;
+        emitDesktopDebug("reconnect.attempt", "device", {}, {
+            platform: "ios",
+            stage: "success",
+            sessionId: session.sessionId
+        });
     }
 }
 /**
@@ -271,6 +304,11 @@ export function getDevicePixelRatio() {
  */
 export async function disconnect() {
     if (activeSession) {
+        emitDesktopDebug("device.disconnect", "device", {}, {
+            platform: "ios",
+            udid: activeSession.udid,
+            simulatorName: activeSession.simulatorName
+        });
         try {
             await appium.deleteSession(activeSession.sessionId);
         }

package/build/src/device/loadmill.js

@@ -2,6 +2,10 @@
  * Loadmill instruction handling for script execution
  */
 import { executeLoadmillCommand } from "../integrations/loadmill/index.js";
+function getLoadmillSiteBaseUrl() {
+    const rawBaseUrl = process.env.LOADMILL_BASE_URL || "https://app.loadmill.com/api";
+    return rawBaseUrl.replace(/\/api\/?$/, "");
+}
 /**
  * Check if an instruction is a Loadmill command
  * @param {string} userInput - The instruction text
@@ -35,22 +39,37 @@ export function extractLoadmillCommand(userInput) {
  * @param {string} command - The Loadmill command to execute
  * @param {boolean} isHeadlessMode - Whether running in headless/CI mode
  * @param {Object} context - Execution context
+ * @param {Object|null} stepContext - Optional step context metadata
  * @returns {Promise<{success: boolean, error?: string}>}
  */
-export async function executeLoadmillInstruction(command, isHeadlessMode, context) {
+export async function executeLoadmillInstruction(command, isHeadlessMode, context, stepContext = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
-    addOutput({ type: 'info', text: `[Loadmill] Executing: ${command}` });
+    const meta = {
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex
+    };
+    addOutput({
+        type: 'info',
+        text: `[Loadmill] Executing: ${command}`,
+        eventType: 'system_message',
+        payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() },
+        ...meta
+    });
     const result = await executeLoadmillCommand(command, {
-        onProgress: ({ message }) => {
-            addOutput({ type: 'info', text: `[Loadmill] ${message}` });
+        onProgress: ({ message, runId }) => {
+            const payload = runId
+                ? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
+                : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
+            addOutput({ type: 'info', text: `[Loadmill] ${message}`, eventType: 'system_message', payload, ...meta });
         }
     });
     if (result.success) {
-        handleLoadmillSuccess(command, result, context);
+        handleLoadmillSuccess(command, result, context, stepContext);
         return { success: true };
     }
     else {
-        return await handleLoadmillFailure(command, result.error, isHeadlessMode, context);
+        return await handleLoadmillFailure(command, result.error, isHeadlessMode, context, stepContext, result.runId);
     }
 }
 /**
@@ -59,13 +78,22 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
  * @param {string} error - Error message
  * @param {boolean} isHeadlessMode - Whether running in headless/CI mode
  * @param {Object} context - Execution context
+ * @param {Object|null} stepContext - Optional step context metadata
  * @returns {Promise<{success: boolean, error?: string}>}
  */
-export async function handleLoadmillFailure(command, error, isHeadlessMode, context) {
+export async function handleLoadmillFailure(command, error, isHeadlessMode, context, stepContext = null, suiteRunId = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
-    addOutput({ type: 'error', text: '[Loadmill] FAILED' });
-    addOutput({ type: 'error', text: `Command: ${command}` });
-    addOutput({ type: 'error', text: `Error: ${error}` });
+    const meta = {
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex
+    };
+    const payload = suiteRunId
+        ? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
+        : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
+    addOutput({ type: 'error', text: '[Loadmill] FAILED', eventType: 'error', payload, ...meta });
+    addOutput({ type: 'error', text: `Command: ${command}`, eventType: 'error', payload, ...meta });
+    addOutput({ type: 'error', text: `Error: ${error}`, eventType: 'error', payload, ...meta });
     if (isHeadlessMode) {
         // Headless mode: exit with error code
         if (context?.exit) {
@@ -74,7 +102,13 @@ export async function handleLoadmillFailure(command, error, isHeadlessMode, cont
         process.exit(1);
     }
     // Interactive mode: ask user what to do
-    addOutput({ type: 'system', text: 'What would you like to do? (retry/skip/stop)' });
+    addOutput({
+        type: 'system',
+        text: 'What would you like to do? (retry/skip/stop)',
+        eventType: 'input_request',
+        payload: { options: ['retry', 'skip', 'stop'], ...payload },
+        ...meta
+    });
     const userChoice = await new Promise((resolve) => {
         if (context?.waitForUserInput) {
             context.waitForUserInput().then(resolve);
@@ -90,7 +124,13 @@ export async function handleLoadmillFailure(command, error, isHeadlessMode, cont
         return { success: false, retry: true };
     }
     else if (choice === 'skip' || choice === 's') {
-        addOutput({ type: 'info', text: 'Skipping failed Loadmill command and continuing...' });
+        addOutput({
+            type: 'info',
+            text: 'Skipping failed Loadmill command and continuing...',
+            eventType: 'system_message',
+            payload,
+            ...meta
+        });
         return { success: true }; // Continue to next instruction
     }
     else {
@@ -103,20 +143,29 @@ export async function handleLoadmillFailure(command, error, isHeadlessMode, cont
  * @param {string} command - The executed command
  * @param {Object} result - The execution result
  * @param {Object} context - Execution context
+ * @param {Object|null} stepContext - Optional step context metadata
  */
-export function handleLoadmillSuccess(command, result, context) {
+export function handleLoadmillSuccess(command, result, context, stepContext = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const meta = {
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex
+    };
+    const payload = result.runId
+        ? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
+        : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
     if (result.action === "search") {
-        addOutput({ type: 'success', text: `[Loadmill] Found ${result.result.flows.length} flow(s)` });
+        addOutput({ type: 'success', text: `[Loadmill] Found ${result.result.flows.length} flow(s)`, eventType: 'system_message', payload, ...meta });
         result.result.flows.forEach((flow, i) => {
             const name = flow.description || flow.name || "Unknown";
-            addOutput({ type: 'info', text: `  ${i + 1}. ${name} (ID: ${flow.id})` });
+            addOutput({ type: 'info', text: `  ${i + 1}. ${name} (ID: ${flow.id})`, eventType: 'system_message', payload, ...meta });
         });
     }
     else {
-        addOutput({ type: 'success', text: `[Loadmill] Flow "${result.flowName}" passed` });
+        addOutput({ type: 'success', text: `[Loadmill] Flow "${result.flowName}" passed`, eventType: 'system_message', payload, ...meta });
         if (result.runId) {
-            addOutput({ type: 'info', text: `  Run ID: ${result.runId}` });
+            addOutput({ type: 'info', text: `  Run ID: ${result.runId}`, eventType: 'system_message', payload, ...meta });
         }
     }
 }