@loadmill/droid-cua 2.14.0 → 2.16.0

package/build/src/device/openai.js

@@ -1,10 +1,25 @@
 import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
-import { createOpenAiClient, resolveRuntimeConfig } from "../core/runtime-config.js";
+import { createOpenAiClient, LOADMILL_MODEL_ALIASES, resolveEffectiveHelperModel, resolveRuntimeConfig, } from "../core/runtime-config.js";
 import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
 import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
 import { logger } from "../utils/logger.js";
 import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
 const cuaDebugTracer = new CuaDebugTracer(logger);
+function resolveOpenAiBaseUrl(client) {
+    return client?.baseURL || client?.baseUrl || client?._options?.baseURL || null;
+}
+function resolveOpenAiEndpointUrl(client, endpointPath) {
+    const baseURL = resolveOpenAiBaseUrl(client);
+    if (!baseURL) {
+        return null;
+    }
+    try {
+        return new URL(endpointPath.replace(/^\/+/, ''), `${String(baseURL).replace(/\/+$/, '')}/`).toString();
+    }
+    catch {
+        return `${String(baseURL).replace(/\/+$/, '')}/${endpointPath.replace(/^\/+/, '')}`;
+    }
+}
 function normalizeMessageContent(content) {
     if (Array.isArray(content)) {
         return content.map((item) => {
@@ -81,7 +96,7 @@ const closeCurrentTabTool = {
 const loadmillTool = {
     type: "function",
     name: "loadmill",
-    description: "Run an explicit Loadmill command. Use this only when the instruction explicitly mentions Loadmill and asks for a Loadmill-specific action.",
+    description: "Search Loadmill for the requested flow, run the best match, and wait for completion. Use this when the instruction explicitly mentions running or using a Loadmill flow.",
     strict: true,
     parameters: {
         type: "object",
@@ -225,14 +240,15 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
  * Revise a test script based on user feedback using simple chat completion
  * @param {{
  *   client: import("openai").default,
+ *   model?: string,
  *   originalScript: string,
  *   revisionRequest: string,
  * }} options
  * @returns {Promise<string>} - The revised test script
  */
-export async function reviseTestScript({ client, originalScript, revisionRequest }) {
+export async function reviseTestScript({ client, model = LOADMILL_MODEL_ALIASES.DEFAULT_AGENT_MODEL, originalScript, revisionRequest, }) {
     const response = await client.chat.completions.create({
-        model: "gpt-5.4",
+        model,
         messages: [{
                 role: "system",
                 content: buildTestRevisionSystemPrompt(originalScript, revisionRequest)
@@ -274,7 +290,14 @@ export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT
         debugContext,
         previousResponseId
     });
+    trace.endpoint = {
+        baseURL: resolveOpenAiBaseUrl(client),
+        responsesURL: resolveOpenAiEndpointUrl(client, "responses"),
+        model: requestParams.model,
+        previousResponseId: requestParams.previous_response_id ?? null,
+    };
     logger.debug("CUA Request:", trace.requestLog);
+    logger.debug("CUA Request endpoint:", trace.endpoint);
     try {
         const response = await client.responses.create(requestParams);
         cuaDebugTracer.onResponse(trace, response);
@@ -311,6 +334,10 @@ export function createOpenAiExecutor({ client, config }) {
         async reviseTestScript(originalScript, revisionRequest) {
             return await reviseTestScript({
                 client: openAiClient,
+                model: resolveEffectiveHelperModel({
+                    llmProviderMode: runtimeConfig.llmProviderMode,
+                    cuaModel: runtimeConfig.cuaModel,
+                }),
                 originalScript,
                 revisionRequest,
             });

package/build/src/integrations/loadmill/executor.js

@@ -4,9 +4,30 @@
 import { createLoadmillApiClient } from "./client.js";
 import { interpretLoadmillCommand, selectBestFlow } from "./interpreter.js";
 import { extractLoadmillRuntimeParameters, renderLoadmillParameterContextNote, selectLoadmillParameterCandidates, } from "./parameters.js";
+import { LOADMILL_MODEL_ALIASES } from "../../core/runtime-config.js";
 import { logger } from "../../utils/logger.js";
 const POLL_INTERVAL_MS = 5000; // 5 seconds
-const TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
+const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes
+function readTimeoutMsFromEnv() {
+    const raw = process.env.LOADMILL_RUN_TIMEOUT_MS;
+    if (!raw) {
+        return DEFAULT_TIMEOUT_MS;
+    }
+    const parsed = Number.parseInt(raw, 10);
+    if (Number.isFinite(parsed) && parsed > 0) {
+        return parsed;
+    }
+    logger.debug("Ignoring invalid LOADMILL_RUN_TIMEOUT_MS value", { value: raw });
+    return DEFAULT_TIMEOUT_MS;
+}
+function formatDuration(ms) {
+    const totalSeconds = Math.ceil(ms / 1000);
+    if (totalSeconds >= 60 && totalSeconds % 60 === 0) {
+        const minutes = totalSeconds / 60;
+        return `${minutes} ${minutes === 1 ? "minute" : "minutes"}`;
+    }
+    return `${totalSeconds} ${totalSeconds === 1 ? "second" : "seconds"}`;
+}
 function buildParameterSelectionMessages({ command, flowName, futureInstructions, parameters, }) {
     const parameterList = parameters
         .map((param) => `- ${param.key}`)
@@ -42,7 +63,7 @@ async function selectRelevantParameterNames({ openAiClient, command, flowName, f
     }
     try {
         const response = await openAiClient.chat.completions.create({
-            model: "gpt-5.4",
+            model: LOADMILL_MODEL_ALIASES.DEFAULT_AGENT_MODEL,
             messages: buildParameterSelectionMessages({
                 command,
                 flowName,
@@ -127,9 +148,10 @@ function sleep(ms) {
  * @param {Function} onStatusUpdate - Callback for status updates
  * @returns {Promise<Object>} - Final run status
  */
-async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => { }) {
-    const startTime = Date.now();
-    while (Date.now() - startTime < TIMEOUT_MS) {
+async function pollForCompletion(runId, loadmillClient, options = {}) {
+    const { onStatusUpdate = () => { }, timeoutMs = readTimeoutMsFromEnv(), pollIntervalMs = POLL_INTERVAL_MS, now = Date.now, sleepFn = sleep, } = options;
+    const startTime = now();
+    while (now() - startTime < timeoutMs) {
         try {
             const status = await loadmillClient.getTestRunStatus(runId);
             onStatusUpdate(status);
@@ -158,7 +180,7 @@ async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => {
                 return { success: false, status: "stopped", result: status, error: "Test flow was stopped" };
             }
             // Still running, wait and poll again
-            await sleep(POLL_INTERVAL_MS);
+            await sleepFn(pollIntervalMs);
         }
         catch (err) {
             logger.error("Error polling Loadmill run status", { runId, error: err.message });
@@ -166,7 +188,7 @@ async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => {
         }
     }
     // Timeout
-    return { success: false, status: "timeout", error: "Test run timed out after 5 minutes" };
+    return { success: false, status: "timeout", error: `Test run timed out after ${formatDuration(timeoutMs)}` };
 }
 /**
  * Execute a Loadmill command from natural language input
@@ -176,7 +198,7 @@ async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => {
  * @returns {Promise<{success: boolean, error?: string, result?: Object}>}
  */
 export async function executeLoadmillCommand(userInput, options = {}) {
-    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, futureInstructions = [], } = options;
+    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, futureInstructions = [], pollTimeoutMs, pollIntervalMs, now, sleepFn, } = options;
     // Check for API token
     if (!loadmillClient.getApiToken()) {
         return {
@@ -214,18 +236,6 @@ export async function executeLoadmillCommand(userInput, options = {}) {
                 error: "Could not select a matching flow"
             };
         }
-        // If action is "search", just return the results
-        if (interpreted.action === "search") {
-            return {
-                success: true,
-                action: "search",
-                result: {
-                    flows,
-                    selectedFlow,
-                    confidence
-                }
-            };
-        }
         // Step 4: Run the flow
         const flowName = selectedFlow.description || selectedFlow.name || "Unknown";
         onProgress({
@@ -244,8 +254,14 @@ export async function executeLoadmillCommand(userInput, options = {}) {
         }
         // Step 5: Poll for completion
         onProgress({ step: "polling", message: `Test started (ID: ${runId}). Waiting for completion...`, runId });
-        const finalResult = await pollForCompletion(runId, loadmillClient, (status) => {
-            onProgress({ step: "polling", message: `Status: ${status.status}...`, runId });
+        const finalResult = await pollForCompletion(runId, loadmillClient, {
+            timeoutMs: pollTimeoutMs,
+            pollIntervalMs,
+            now,
+            sleepFn,
+            onStatusUpdate: (status) => {
+                onProgress({ step: "polling", message: `Status: ${status.status}...`, runId });
+            },
         });
         const runtimeParameterContext = finalResult.success
             ? await buildRuntimeParameterContext({

package/build/src/integrations/loadmill/interpreter.js

@@ -1,15 +1,16 @@
 /**
  * AI-powered text interpretation for Loadmill commands
  */
+import { LOADMILL_MODEL_ALIASES } from "../../core/runtime-config.js";
 import { buildLoadmillCommandInterpretationMessages, buildLoadmillFlowSelectionMessages, } from "../../prompts/loadmill.js";
 /**
  * Interpret a natural language Loadmill command into structured data
  * @param {string} userInput - Natural language command
- * @returns {Promise<{searchQuery: string, parameters: Object, action: 'run'|'search'}>}
+ * @returns {Promise<{searchQuery: string, parameters: Object}>}
  */
 export async function interpretLoadmillCommand(userInput, { client }) {
     const response = await client.chat.completions.create({
-        model: "gpt-4o-mini",
+        model: LOADMILL_MODEL_ALIASES.FAST,
         messages: buildLoadmillCommandInterpretationMessages(userInput),
         response_format: { type: "json_object" }
     });
@@ -18,7 +19,6 @@ export async function interpretLoadmillCommand(userInput, { client }) {
     return {
         searchQuery: parsed.searchQuery || userInput,
         parameters: parsed.parameters || {},
-        action: parsed.action || "run"
     };
 }
 /**
@@ -42,7 +42,7 @@ export async function selectBestFlow(flows, originalQuery, { client }) {
         return `${i + 1}. ID: ${f.id}, Name: "${name}"${suite ? `, Suite: "${suite}"` : ""}`;
     }).join("\n");
     const response = await client.chat.completions.create({
-        model: "gpt-4o-mini",
+        model: LOADMILL_MODEL_ALIASES.FAST,
         messages: buildLoadmillFlowSelectionMessages(originalQuery, flowList),
         response_format: { type: "json_object" }
     });

package/build/src/modes/execution-mode.js

@@ -8,6 +8,31 @@ import { isAssertion } from "../device/assertions.js";
 import { logger } from "../utils/logger.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { printCliOutput } from "../utils/console-output.js";
+// Mirrors the codes emitted by the Loadmill LLM gateway (see services/app/private/llm
+// in the server repo): a transient burst limit plus day/week/fixed-budget request and
+// token quotas. Free plans run on the fixed-budget quota.
+const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
+    "llm_rate_limit_exceeded",
+    "llm_day_quota_exceeded",
+    "llm_week_quota_exceeded",
+    "llm_fixed_budget_quota_exceeded",
+    "llm_day_token_quota_exceeded",
+    "llm_week_token_quota_exceeded",
+    "llm_fixed_budget_token_quota_exceeded",
+]);
+const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
+function isLlmUsageLimitError(err) {
+    if (!err || typeof err !== "object") {
+        return false;
+    }
+    if (typeof err.code === "string" && LLM_USAGE_LIMIT_ERROR_CODES.has(err.code)) {
+        return true;
+    }
+    return err.status === 429 && err.type === "rate_limit_error";
+}
+function getLlmUsageLimitMessage() {
+    return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
+}
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -267,6 +292,26 @@ export class ExecutionMode {
                 stack: err.stack
             });
             const addOutput = context.addOutput || printCliOutput;
+            if (isLlmUsageLimitError(err)) {
+                const stopMessage = getLlmUsageLimitMessage();
+                logger.debug('Stopping execution because an LLM usage limit was reached', {
+                    instruction,
+                    message: err.message,
+                    status: err.status,
+                    code: err.code,
+                    type: err.type,
+                });
+                this.emit(addOutput, 'error', stopMessage, context, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: stopMessage,
+                        status: err.status,
+                        code: err.code,
+                        type: err.type,
+                    }
+                });
+                return { success: false, error: stopMessage, usageLimitReached: true };
+            }
             // Check if we've exceeded max retries
             if (retryCount >= MAX_RETRIES) {
                 emitDesktopDebug("reconnect.attempt", "device", {

package/build/src/prompts/execution.js

@@ -44,7 +44,7 @@ Your process:
    - Before calling \`assert\`, make sure the current screen contains the evidence needed to validate the condition. If needed, perform only minimal preparation first, such as taking a screenshot, waiting briefly, closing the keyboard, scrolling within the relevant view, or opening the relevant menu.
    - Once you call \`assert\`, validation is read-only. The assertion tool will not click, type, wait, scroll, request screenshots, or change the device state.
    - Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
-   - Use \`loadmill\` only when the instruction explicitly references the word "loadmill"
+   - Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
 3. Execute the required actions
 4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
 5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation

package/build/src/prompts/loadmill.js

@@ -7,25 +7,21 @@ export function buildLoadmillCommandInterpretationMessages(userInput) {
 Extract the following from the user's input:
 1. searchQuery: The flow name or description to search for (required). FIX any obvious typos or misspellings.
 2. parameters: Any key=value pairs mentioned (as an object)
-3. action: Either "run" (if user wants to execute) or "search" (if user just wants to find flows)
 
 Output JSON only, no markdown or explanation.
 
 Examples:
 Input: "run the checkout flow with user=test123"
-Output: {"searchQuery": "checkout flow", "parameters": {"user": "test123"}, "action": "run"}
-
-Input: "search for login test"
-Output: {"searchQuery": "login test", "parameters": {}, "action": "search"}
+Output: {"searchQuery": "checkout flow", "parameters": {"user": "test123"}}
 
 Input: "run user authentication with email=test@example.com password=secret123"
-Output: {"searchQuery": "user authentication", "parameters": {"email": "test@example.com", "password": "secret123"}, "action": "run"}
+Output: {"searchQuery": "user authentication", "parameters": {"email": "test@example.com", "password": "secret123"}}
 
 Input: "execute payment flow"
-Output: {"searchQuery": "payment flow", "parameters": {}, "action": "run"}
+Output: {"searchQuery": "payment flow", "parameters": {}}
 
 Input: "create a transction with amount=200"
-Output: {"searchQuery": "transaction", "parameters": {"amount": "200"}, "action": "run"}`
+Output: {"searchQuery": "transaction", "parameters": {"amount": "200"}}`
         },
         {
             role: "user",

package/build/src/utils/cua-debug-tracer.js

@@ -23,6 +23,17 @@ function safeHeadersSnapshot(headers) {
     }
     return out;
 }
+function summarizeErrorCause(error) {
+    const cause = error?.cause;
+    if (!cause || typeof cause !== "object") {
+        return null;
+    }
+    return {
+        name: typeof cause.name === "string" ? cause.name : null,
+        message: typeof cause.message === "string" ? cause.message : String(cause),
+        code: typeof cause.code === "string" ? cause.code : null,
+    };
+}
 function summarizeContent(content) {
     if (typeof content === "string") {
         return truncateForDebug(content, 1600);
@@ -394,6 +405,7 @@ export class CuaDebugTracer {
             : null;
         const requestId = err?.request_id || responseError?.request_id || requestIdFromHeaders || null;
         const headers = safeHeadersSnapshot(err?.headers);
+        const provider = typeof responseError?.provider === "string" ? responseError.provider : null;
         emitDesktopDebug("device.error", "device", trace.ids, {
             localRequestId: trace.localRequestId,
             operation: "cua.request",
@@ -403,8 +415,11 @@ export class CuaDebugTracer {
             code: err?.code ?? responseError?.code ?? null,
             type: responseError?.type ?? null,
             param: responseError?.param ?? null,
+            provider,
             requestId,
             headers,
+            endpoint: trace.endpoint || null,
+            cause: summarizeErrorCause(err),
             chain: {
                 lastResponseId: trace.lastResponseMeta?.id ?? null,
                 lastResponseOutputTypes: trace.lastResponseMeta?.outputTypes ?? [],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.14.0",
+  "version": "2.16.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -15,15 +15,19 @@
   ],
   "scripts": {
     "start": "tsx index.js",
-    "bench": "npm run bench:preflight && npm run bench:run && npm run bench:report",
+    "bench": "node scripts/benchmark.js",
     "bench:preflight": "node scripts/benchmark-preflight.js",
     "bench:run": "tsx index.js --platform android --instructions benchmarks/benchmark.dcua --context benchmarks/context.md --debug",
     "bench:report": "node scripts/benchmark-report.js",
+    "bench:report:web": "node scripts/benchmark-report-web.js",
     "execution:profile": "node scripts/execution-profile.js",
     "test": "node --test unit-tests/*.test.js",
     "build": "tsc",
     "clean": "rm -rf build",
     "typecheck": "tsc --noEmit",
+    "typecheck:desktop": "npm --prefix desktop-app run typecheck",
+    "typecheck:all": "npm run typecheck && npm run typecheck:desktop",
+    "prepare": "node scripts/install-git-hooks.mjs",
     "prepublishOnly": "npm run build"
   },
   "repository": {