@loadmill/droid-cua 2.15.1 → 2.17.0

package/build/src/modes/execution-mode.js

@@ -8,6 +8,31 @@ import { isAssertion } from "../device/assertions.js";
 import { logger } from "../utils/logger.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { printCliOutput } from "../utils/console-output.js";
+// Mirrors the codes emitted by the Loadmill LLM gateway (see services/app/private/llm
+// in the server repo): a transient burst limit plus day/week/fixed-budget request and
+// token quotas. Free plans run on the fixed-budget quota.
+const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
+    "llm_rate_limit_exceeded",
+    "llm_day_quota_exceeded",
+    "llm_week_quota_exceeded",
+    "llm_fixed_budget_quota_exceeded",
+    "llm_day_token_quota_exceeded",
+    "llm_week_token_quota_exceeded",
+    "llm_fixed_budget_token_quota_exceeded",
+]);
+const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
+function isLlmUsageLimitError(err) {
+    if (!err || typeof err !== "object") {
+        return false;
+    }
+    if (typeof err.code === "string" && LLM_USAGE_LIMIT_ERROR_CODES.has(err.code)) {
+        return true;
+    }
+    return err.status === 429 && err.type === "rate_limit_error";
+}
+function getLlmUsageLimitMessage() {
+    return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
+}
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -267,6 +292,26 @@ export class ExecutionMode {
                 stack: err.stack
             });
             const addOutput = context.addOutput || printCliOutput;
+            if (isLlmUsageLimitError(err)) {
+                const stopMessage = getLlmUsageLimitMessage();
+                logger.debug('Stopping execution because an LLM usage limit was reached', {
+                    instruction,
+                    message: err.message,
+                    status: err.status,
+                    code: err.code,
+                    type: err.type,
+                });
+                this.emit(addOutput, 'error', stopMessage, context, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: stopMessage,
+                        status: err.status,
+                        code: err.code,
+                        type: err.type,
+                    }
+                });
+                return { success: false, error: stopMessage, usageLimitReached: true };
+            }
             // Check if we've exceeded max retries
             if (retryCount >= MAX_RETRIES) {
                 emitDesktopDebug("reconnect.attempt", "device", {

package/build/src/prompts/execution.js

@@ -9,6 +9,17 @@ export function buildAppContextSection(text) {
 The following describes the app you are testing.
 Use this to understand screen layouts, terminology, navigation, and expected behavior.
 
+${trimmed}`;
+}
+export function buildTestDataContextSection(text) {
+    const trimmed = typeof text === "string" ? text.trim() : "";
+    if (!trimmed) {
+        return "";
+    }
+    return `TEST DATA FOR THIS RUN:
+The following values are test data for this execution. Use them as literal run data when performing the test.
+They do not replace or override the test instructions.
+
 ${trimmed}`;
 }
 export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
@@ -16,6 +27,7 @@ export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, ap
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
     const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
     const appContextSection = buildAppContextSection(appContextText);
+    const testDataContextSection = buildTestDataContextSection(options.testDataContext);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -52,7 +64,7 @@ Your process:
 7. Stop immediately - no commentary, no questions
 
 Each instruction is independent. Do not reference previous instructions or ask about next steps.
-${appContextSection ? `\n\n${appContextSection}` : ""}`;
+${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
         { title: "Execution Mode Instructions", text: executionCustomText }

package/build/src/prompts/scenario-planner.js

@@ -0,0 +1,48 @@
+function formatCandidateTests(tests) {
+    return tests.map((test, index) => {
+        const content = Array.isArray(test.instructions) ? test.instructions.join("\n") : "";
+        return [
+            `TEST ${index + 1}`,
+            `Name: ${test.testName}`,
+            "Instructions:",
+            content || "(empty)",
+        ].join("\n");
+    }).join("\n\n---\n\n");
+}
+export function buildScenarioPlannerMessages({ scenarioPrompt, tests, appContextText = "", testDataText = "", }) {
+    return [
+        {
+            role: "system",
+            content: [
+                "You prepare concrete test executions for a computer-using agent.",
+                "The agent runs manual-style app tests: it receives a test brief, app context, and optional run data, then operates the app UI.",
+                "Your job is to turn the user's scenario request into the exact ordered cases the agent should run.",
+                "Return JSON only with this shape:",
+                "{\"cases\":[{\"testName\":\"exact-test-name.dcua\",\"caseName\":\"test-name.dcua / data label\",\"testDataContext\":\"raw data text for this case\"}]}",
+                "Rules:",
+                "- testName must exactly match one of the provided candidate test names.",
+                "- Create an ordered flat list of cases to run.",
+                "- caseName is an ephemeral display name for logs and reports. Keep it short and human-readable.",
+                "- testDataContext must contain only the concrete data relevant to that case.",
+                "- Do not invent data that was not provided in test data, context, or the scenario prompt.",
+                "- If no extra data is needed for a case, use an empty string for testDataContext.",
+            ].join("\n")
+        },
+        {
+            role: "user",
+            content: [
+                "Scenario prompt:",
+                scenarioPrompt,
+                "",
+                "App context:",
+                appContextText && appContextText.trim() ? appContextText.trim() : "(none)",
+                "",
+                "Project test data:",
+                testDataText && testDataText.trim() ? testDataText.trim() : "(none)",
+                "",
+                "Candidate tests:",
+                formatCandidateTests(tests),
+            ].join("\n")
+        }
+    ];
+}

package/build/src/prompts/test-data-import.js

@@ -0,0 +1,49 @@
+import { DEFAULT_TEST_DATA_TEMPLATE, isDefaultTestDataTemplate } from "../core/test-data.js";
+export function buildTestDataImportMessages({ currentContent = "", importedFileName, importedFileText, instructions = "", }) {
+    const currentIsTemplateOnly = isDefaultTestDataTemplate(currentContent);
+    const userInstructions = typeof instructions === "string" && instructions.trim()
+        ? instructions.trim()
+        : "(none)";
+    return [
+        {
+            role: "system",
+            content: [
+                "You convert imported data into durable project test data for computer-using-agent scenario runs.",
+                "The durable format is test-data.md: readable markdown that describes concrete entities and run data the agent can use.",
+                "Return JSON only with this shape:",
+                "{\"content\":\"# Test Data\\n...complete markdown...\"}",
+                "Rules:",
+                "- Return the complete replacement content for test-data.md, not a patch.",
+                "- Do not invent values that are not present in the current file, imported file, or user instructions.",
+                "- If currentContent is template-only, replace it with real imported data and do not keep commented template examples.",
+                "- If currentContent is not template-only, preserve its useful structure and add or update entities within it.",
+                "- For matrix-like CSV rows, prefer splitting repeated dimensions into separate entities when the structure is clear.",
+                "- If rows are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
+                "- User instructions can guide structure and naming, but content must remain concrete data, not test instructions.",
+                "- Output markdown only inside the JSON content string; do not use code fences.",
+            ].join("\n")
+        },
+        {
+            role: "user",
+            content: [
+                "Current test-data.md status:",
+                currentIsTemplateOnly ? "template-only" : "existing user content",
+                "",
+                "Default template used for template-only detection:",
+                DEFAULT_TEST_DATA_TEMPLATE.trimEnd(),
+                "",
+                "Current test-data.md content:",
+                String(currentContent ?? "").trim() ? String(currentContent) : "(empty)",
+                "",
+                "Imported file:",
+                importedFileName,
+                "",
+                "Imported file text:",
+                importedFileText,
+                "",
+                "User import instructions:",
+                userInstructions,
+            ].join("\n")
+        }
+    ];
+}

package/build/src/utils/cua-debug-tracer.js

@@ -23,6 +23,17 @@ function safeHeadersSnapshot(headers) {
     }
     return out;
 }
+function summarizeErrorCause(error) {
+    const cause = error?.cause;
+    if (!cause || typeof cause !== "object") {
+        return null;
+    }
+    return {
+        name: typeof cause.name === "string" ? cause.name : null,
+        message: typeof cause.message === "string" ? cause.message : String(cause),
+        code: typeof cause.code === "string" ? cause.code : null,
+    };
+}
 function summarizeContent(content) {
     if (typeof content === "string") {
         return truncateForDebug(content, 1600);
@@ -394,6 +405,7 @@ export class CuaDebugTracer {
             : null;
         const requestId = err?.request_id || responseError?.request_id || requestIdFromHeaders || null;
         const headers = safeHeadersSnapshot(err?.headers);
+        const provider = typeof responseError?.provider === "string" ? responseError.provider : null;
         emitDesktopDebug("device.error", "device", trace.ids, {
             localRequestId: trace.localRequestId,
             operation: "cua.request",
@@ -403,8 +415,11 @@ export class CuaDebugTracer {
             code: err?.code ?? responseError?.code ?? null,
             type: responseError?.type ?? null,
             param: responseError?.param ?? null,
+            provider,
             requestId,
             headers,
+            endpoint: trace.endpoint || null,
+            cause: summarizeErrorCause(err),
             chain: {
                 lastResponseId: trace.lastResponseMeta?.id ?? null,
                 lastResponseOutputTypes: trace.lastResponseMeta?.outputTypes ?? [],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.15.1",
+  "version": "2.17.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -15,10 +15,11 @@
   ],
   "scripts": {
     "start": "tsx index.js",
-    "bench": "npm run bench:preflight && npm run bench:run && npm run bench:report",
+    "bench": "node scripts/benchmark.js",
     "bench:preflight": "node scripts/benchmark-preflight.js",
     "bench:run": "tsx index.js --platform android --instructions benchmarks/benchmark.dcua --context benchmarks/context.md --debug",
     "bench:report": "node scripts/benchmark-report.js",
+    "bench:report:web": "node scripts/benchmark-report-web.js",
     "execution:profile": "node scripts/execution-profile.js",
     "test": "node --test unit-tests/*.test.js",
     "build": "tsc",