@loadmill/droid-cua 2.34.0 → 2.36.0

package/build/src/modes/execution-mode.js

@@ -10,6 +10,8 @@ import { handleAssertionToolCall } from "../core/tool-handlers/assert.js";
 import { logger } from "../utils/logger.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { printCliOutput } from "../utils/console-output.js";
+import { APIConnectionError } from "openai";
+import { findDeepestErrorCode } from "../utils/error-details.js";
 // Mirrors the quota codes emitted by the Loadmill LLM gateway (see
 // services/app/private/llm in the server repo). Free plans run on the
 // fixed-budget quota.
@@ -50,6 +52,13 @@ function isRejectedContinuationError(err) {
         "invalid_previous_response_id",
     ].includes(err.code);
 }
+function isLlmConnectionError(err) {
+    return err instanceof APIConnectionError || err?.name === "APIConnectionError" || err?.name === "APIConnectionTimeoutError";
+}
+function getLlmConnectionErrorMessage(err) {
+    const networkCode = findDeepestErrorCode(err);
+    return `Could not connect to the AI service${networkCode ? ` (${networkCode})` : ""}. Check your network, proxy, or security software and try again.`;
+}
 function getLlmUsageLimitMessage() {
     return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
 }
@@ -422,6 +431,22 @@ export class ExecutionMode {
                 stack: err.stack
             });
             const addOutput = context.addOutput || printCliOutput;
+            if (isLlmConnectionError(err)) {
+                const stopMessage = getLlmConnectionErrorMessage(err);
+                logger.debug('Stopping execution after an LLM connection failure', {
+                    instruction,
+                    message: err.message,
+                    code: findDeepestErrorCode(err),
+                });
+                this.emit(addOutput, 'error', stopMessage, context, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: stopMessage,
+                        code: findDeepestErrorCode(err),
+                    }
+                });
+                return { success: false, error: stopMessage };
+            }
             if (this.session.previousResponseId && isRejectedContinuationError(err)) {
                 this.emit(addOutput, 'error', err.message, context, stepContext, {
                     eventType: 'error',

package/build/src/prompts/base.js

@@ -30,6 +30,9 @@ ${customSection}
 function isBrowserTarget(deviceInfo = {}) {
     return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
 }
+function isDesktopTarget(deviceInfo = {}) {
+    return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "desktop";
+}
 function isCloudIosTarget(deviceInfo = {}) {
     const platform = deviceInfo.platform?.trim().toLowerCase();
     const source = deviceInfo.source?.trim().toLowerCase() || "local";
@@ -42,6 +45,9 @@ function describeControlledTarget(deviceInfo = {}) {
     if (platform === "browser") {
         return browserName ? `a browser session (${browserName})` : "a browser session";
     }
+    if (platform === "desktop") {
+        return deviceName ? `a desktop application (${deviceName})` : "a desktop application";
+    }
     if (platform === "ios") {
         const target = isCloudIosTarget(deviceInfo) ? "an iOS device" : "an iOS simulator";
         return deviceName ? `${target} (${deviceName})` : target;
@@ -248,9 +254,37 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
   - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
   `;
 }
+function buildDesktopBaseSystemPrompt(deviceInfo, options = {}) {
+    let browserPrompt = buildBrowserBaseSystemPrompt(deviceInfo, options)
+        .replace("Follow the user's instructions to interact with the browser.", "Follow the user's instructions to interact with the selected application.")
+        .replace("visible browser window", "selected application window")
+        .replace("CRITICAL - Desktop Browser Input Constraints:", "CRITICAL - Desktop Input Constraints:")
+        .replace("This is a desktop browser environment.", "This is a desktop application environment.");
+    if (deviceInfo.desktopInputMode !== "physical") {
+        browserPrompt = browserPrompt
+            .replace("Available actions: click, scroll, type, keypress, wait, screenshot.", "Available actions: click, scroll, wait, screenshot.")
+            .replace("Standard desktop keys and modifier chords are allowed when they are clearly useful.", "Keyboard input is unavailable because it would focus the application.")
+            .replace("Prefer clicking visible controls over keyboard shortcuts when both are equally reliable.", "Use visible application controls for every interaction.")
+            .replace("Use keypress chords only when they directly help the task, such as Enter to submit, Escape to dismiss a modal, or Control/Meta shortcuts for focused text fields.", "Use visible application controls for submit, dismiss, and correction actions.");
+        browserPrompt = browserPrompt.replace("After EVERY action (click, type, keypress, scroll)", "After EVERY action (click, scroll)");
+    }
+    const notesIndex = browserPrompt.indexOf("  Browser-Specific Notes:");
+    const withoutBrowserNotes = notesIndex === -1 ? browserPrompt : browserPrompt.slice(0, notesIndex);
+    const inputNote = deviceInfo.desktopInputMode === "physical"
+        ? "Droid may use the physical OS pointer for this run"
+        : "Cursor-free mode never focuses the application: use only clicks and scrolling on controls with matching accessibility actions; never emit type, keypress, move, hover, or drag";
+    return `${withoutBrowserNotes}  Desktop-Specific Notes:
+  - Droid is controlling only the selected application window, not the whole display
+  - ${inputNote}
+  - Do not use shortcuts that switch applications, virtual desktops, or operating-system context
+  - Dialogs owned by the selected application may be controlled when they become its active window
+  `;
+}
 function buildBaseSystemPrompt(deviceInfo, options = {}) {
-    return isBrowserTarget(deviceInfo)
-        ? buildBrowserBaseSystemPrompt(deviceInfo, options)
-        : buildMobileBaseSystemPrompt(deviceInfo, options);
+    if (isBrowserTarget(deviceInfo))
+        return buildBrowserBaseSystemPrompt(deviceInfo, options);
+    if (isDesktopTarget(deviceInfo))
+        return buildDesktopBaseSystemPrompt(deviceInfo, options);
+    return buildMobileBaseSystemPrompt(deviceInfo, options);
 }
-export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, };
+export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, isDesktopTarget, };

package/build/src/prompts/test-data-import.js

@@ -17,8 +17,8 @@ export function buildTestDataImportMessages({ currentContent = "", importedFileN
                 "- Do not invent values that are not present in the current file, imported file, or user instructions.",
                 "- If currentContent is template-only, replace it with real imported data and do not keep commented template examples.",
                 "- If currentContent is not template-only, preserve its useful structure and add or update entities within it.",
-                "- For matrix-like CSV rows, prefer splitting repeated dimensions into separate entities when the structure is clear.",
-                "- If rows are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
+                "- For tabular or nested data, preserve meaningful groups and relationships. Split repeated dimensions into separate entities when the structure is clear.",
+                "- If records are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
                 "- User instructions can guide structure and naming, but content must remain concrete data, not test instructions.",
                 "- Output markdown only inside the JSON content string; do not use code fences.",
             ].join("\n")

package/build/src/utils/cua-debug-tracer.js

@@ -1,5 +1,6 @@
 import crypto from "crypto";
 import { emitDesktopDebug, truncateForDebug } from "./desktop-debug.js";
+import { summarizeError } from "./error-details.js";
 function safeHeadersSnapshot(headers) {
     if (!headers || typeof headers !== "object")
         return {};
@@ -23,17 +24,6 @@ function safeHeadersSnapshot(headers) {
     }
     return out;
 }
-function summarizeErrorCause(error) {
-    const cause = error?.cause;
-    if (!cause || typeof cause !== "object") {
-        return null;
-    }
-    return {
-        name: typeof cause.name === "string" ? cause.name : null,
-        message: typeof cause.message === "string" ? cause.message : String(cause),
-        code: typeof cause.code === "string" ? cause.code : null,
-    };
-}
 function summarizeContent(content) {
     if (typeof content === "string") {
         return truncateForDebug(content, 1600);
@@ -462,7 +452,7 @@ export class CuaDebugTracer {
             requestId,
             headers,
             endpoint: trace.endpoint || null,
-            cause: summarizeErrorCause(err),
+            cause: summarizeError(err?.cause),
             chain: {
                 lastResponseId: trace.lastResponseMeta?.id ?? null,
                 lastResponseOutputTypes: trace.lastResponseMeta?.outputTypes ?? [],

package/build/src/utils/error-details.js

@@ -0,0 +1,56 @@
+const ERROR_FIELDS = [
+    "name",
+    "message",
+    "code",
+    "errno",
+    "syscall",
+    "hostname",
+    "address",
+    "port",
+];
+export function summarizeError(error, maxCauseDepth = 5) {
+    if (error == null) {
+        return null;
+    }
+    if (typeof error !== "object") {
+        return { message: String(error) };
+    }
+    const seen = new Set();
+    function visit(current, depth) {
+        if (!current || typeof current !== "object" || seen.has(current)) {
+            return null;
+        }
+        seen.add(current);
+        const summary = {};
+        for (const field of ERROR_FIELDS) {
+            const value = current[field];
+            if (typeof value === "string" || typeof value === "number") {
+                summary[field] = value;
+            }
+        }
+        if (!summary.message) {
+            summary.message = String(current);
+        }
+        if (depth < maxCauseDepth) {
+            const cause = visit(current.cause, depth + 1);
+            if (cause) {
+                summary.cause = cause;
+            }
+        }
+        return summary;
+    }
+    return visit(error, 0);
+}
+export function findDeepestErrorCode(error) {
+    let current = error;
+    let code = null;
+    const seen = new Set();
+    while (current && typeof current === "object" && !seen.has(current)) {
+        seen.add(current);
+        if (typeof current.code === "string" || typeof current.code === "number") {
+            code = String(current.code);
+        }
+        current = current.cause;
+    }
+    return code;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.34.0",
+  "version": "2.36.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -12,6 +12,7 @@
     "bin/**/*",
     "skill/**/*",
     "README.md",
+    "THIRD_PARTY_NOTICES.md",
     "LICENSE"
   ],
   "scripts": {
@@ -63,6 +64,7 @@
   "license": "UNLICENSED",
   "dependencies": {
     "@aws-sdk/client-device-farm": "^3.1065.0",
+    "@zavora-ai/computer-use-mcp": "7.1.0",
     "dotenv": "^16.4.7",
     "ink": "^4.4.1",
     "ink-spinner": "^5.0.0",
@@ -78,6 +80,6 @@
     "typescript": "^5.7.3"
   },
   "engines": {
-    "node": ">=18.17.0"
+    "node": ">=20.0.0"
   }
 }

package/skill/droid-cua/references/running-and-healing.md

@@ -2,7 +2,11 @@
 
 ## Construct the run
 
-Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider with Loadmill Pulse as the recommended model for daily authoring, local runs, and CI. Escalate to Loadmill Smart when stability is a demonstrated problem and run evidence shows Smart improves it:
+Prefer a single test before a directory, multi-path selection, or scenario run. Use `droid-cua run <paths...>` for saved tests. It accepts one or more `.dcua` files or directories, including files from different folders. `run` must be the first argument. Keep all selected paths contiguous and place options after the complete path list. Inputs retain command-line order, directories use natural test-name ordering, and overlapping files run only once. The legacy `--instructions <file-or-directory>` form remains supported, but prefer `run` for new commands.
+
+A single target, config, context, test-data file, and secrets source applies to the entire command. Combine tests only when they share those run-level settings; otherwise use separate commands. Multi-path runs execute sequentially and produce an aggregate **Selected tests** report in addition to individual test reports.
+
+Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider with Loadmill Pulse as the recommended model for daily authoring, local runs, and CI. Escalate to Loadmill Smart when stability is a demonstrated problem and run evidence shows Smart improves it:
 
 ```sh
 # Recommended daily model
@@ -18,25 +22,28 @@ Include `--report reports/<test-name>.html` for normal test runs unless the user
 
 ```sh
 # Attached Android device or running emulator
-droid-cua --llm-provider loadmill --cua-model loadmill-pulse --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
+droid-cua run tests/example.dcua --llm-provider loadmill --cua-model loadmill-pulse --avd adb:<serial> --context tests/context.md --debug --report reports/example.html
 
 # Launchable Android AVD
-droid-cua --avd avd:<avd-name> --instructions tests/example.dcua --config ci/droid-cua.json --debug --report reports/example.html
+droid-cua run tests/example.dcua --avd avd:<avd-name> --config ci/droid-cua.json --debug --report reports/example.html
 
 # iOS Simulator
-droid-cua --llm-provider loadmill --cua-model loadmill-pulse --platform ios --avd "<simulator-name>" --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua run tests/example.dcua --llm-provider loadmill --cua-model loadmill-pulse --platform ios --avd "<simulator-name>" --debug --report reports/example.html
 
 # Local web
-droid-cua --llm-provider loadmill --cua-model loadmill-pulse --target web --browser chrome --session-mode fresh --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua run tests/example.dcua --llm-provider loadmill --cua-model loadmill-pulse --target web --browser chrome --session-mode fresh --debug --report reports/example.html
 
 # LambdaTest
-droid-cua --llm-provider loadmill --cua-model loadmill-pulse --device-source lambdatest --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua run tests/example.dcua --llm-provider loadmill --cua-model loadmill-pulse --device-source lambdatest --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --debug --report reports/example.html
 
 # AWS Device Farm
-droid-cua --llm-provider loadmill --cua-model loadmill-pulse --device-source aws-device-farm --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --instructions tests/example.dcua --debug --report reports/example.html
+droid-cua run tests/example.dcua --llm-provider loadmill --cua-model loadmill-pulse --device-source aws-device-farm --platform android --device-name "<name>" --os-version "<version>" --app <app.apk> --debug --report reports/example.html
 
 # Direct OpenAI alternative
-droid-cua --llm-provider openai --cua-model gpt-5.4 --avd adb:<serial> --instructions tests/example.dcua --context tests/context.md --debug --report reports/example.html
+droid-cua run tests/example.dcua --llm-provider openai --cua-model gpt-5.4 --avd adb:<serial> --context tests/context.md --debug --report reports/example.html
+
+# Selected tests from different folders
+droid-cua run tests/auth/login.dcua tests/checkout/purchase.dcua admin/refund.dcua --avd adb:<serial> --config ci/droid-cua.json --debug --report reports/selected-tests.html
 ```
 
 Use `.ipa` and `--platform ios` for supported cloud iOS runs. Preserve existing config flags rather than replacing them with defaults.