@loadmill/droid-cua 2.25.0 → 2.27.0

package/build/src/device/cloud/progress.js

@@ -7,6 +7,15 @@
  *               previous phase (if any) just ended
  *   - "note":   persistent information worth keeping after the line clears
  *               (e.g. billed device-minutes)
+ *   - "phase-close-text": overrides the persisted "✔ <name> (Nms)" line that
+ *               will be printed when the current phase closes, with an
+ *               arbitrary caller-supplied string (e.g. richer poll/download
+ *               summaries) instead of the default elapsed-ms text.
+ *   - "print":  a verbatim persistent line (no ℹ/✔ prefix) that first closes
+ *               any still-open phase, so a final summary can never race
+ *               ahead of / glue onto the active spinner or an unflushed
+ *               phase-close line. Use this instead of raw console.log while
+ *               a status line may be active.
  * With no listener installed, reports are dropped.
  */
 let listener = null;
@@ -22,3 +31,23 @@ export function reportCloudPhase(name) {
 export function reportCloudNote(message) {
     listener?.({ kind: "note", message });
 }
+export function reportCloudPhaseCloseText(text) {
+    listener?.({ kind: "phase-close-text", message: text });
+}
+// Forces the currently-open phase (if any) to flush its "✔ ..." close line
+// immediately, instead of waiting for the next reportCloudPhase() call to
+// lazily trigger it. Needed when a note must appear strictly *after* a
+// phase's close line rather than racing ahead of it (the close line only
+// carries its final text via reportCloudPhaseCloseText, which by itself
+// doesn't print anything until something forces the phase closed).
+export function reportCloudPhaseFlush() {
+    listener?.({ kind: "phase-flush" });
+}
+export function reportCloudPrint(text) {
+    if (listener) {
+        listener({ kind: "print", message: text });
+    }
+    else {
+        console.log(text);
+    }
+}

package/build/src/device/cloud/registry.js

@@ -1,7 +1,8 @@
 import { browserStackAdapter } from "./browserstack/adapter.js";
 import { lambdaTestAdapter } from "./lambdatest/adapter.js";
 import { awsDeviceFarmAdapter } from "./aws-device-farm/adapter.js";
-const availableAdapters = [browserStackAdapter, lambdaTestAdapter, awsDeviceFarmAdapter];
+import { loadmillCloudAdapter } from "./loadmill-cloud/adapter.js";
+const availableAdapters = [browserStackAdapter, lambdaTestAdapter, awsDeviceFarmAdapter, loadmillCloudAdapter];
 export function listCloudProviderAdapters() {
     return availableAdapters;
 }

package/build/src/device/connection.js

@@ -23,6 +23,9 @@ export async function connectToDevice(deviceNameOrOptions, platform = null, opti
         log(`Platform: ${detectedPlatform}`);
         return currentBackend.connectToDevice(cloudOptions, options);
     }
+    if (getCurrentSource() !== "local" && currentBackend) {
+        throw new Error("A cloud device session is active; disconnect it before connecting to a local device.");
+    }
     const detectedPlatform = platform || detectPlatform(deviceNameOrOptions);
     setCurrentSource("local");
     setCurrentPlatform(detectedPlatform);

package/build/src/device/openai.js

@@ -9,6 +9,7 @@ const cuaDebugTracer = new CuaDebugTracer(logger);
 const LOADMILL_DEVICE_METADATA_MODELS = new Set([
     ...LOADMILL_CUA_MODELS,
     LOADMILL_MODEL_ALIASES.SMART,
+    LOADMILL_MODEL_ALIASES.PULSE,
     LOADMILL_MODEL_ALIASES.BEACON,
 ]);
 function resolveOpenAiBaseUrl(client) {

package/build/src/integrations/loadmill/client.js

@@ -138,6 +138,18 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
             body: JSON.stringify(payload),
         });
     }
+    async function requestDroidRunReportUploadPolicy(uploadId, payload) {
+        return await apiRequest(`/droid-cua/report-uploads/${uploadId}/policy`, {
+            method: "POST",
+            body: JSON.stringify(payload),
+        });
+    }
+    async function associateDroidRunReport(runId, payload) {
+        return await apiRequest(`/droid-cua/runs/${runId}/report`, {
+            method: "POST",
+            body: JSON.stringify(payload),
+        });
+    }
     async function validateApiToken() {
         const resolvedToken = getApiToken();
         if (!resolvedToken) {
@@ -187,6 +199,8 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
         getTestSuiteFlowRun,
         explainFailure,
         reportDroidRun,
+        requestDroidRunReportUploadPolicy,
+        associateDroidRunReport,
         validateApiToken,
         getCurrentUser,
     };

package/build/src/integrations/loadmill/report-upload-manager.js

@@ -0,0 +1,221 @@
+import { createHash, randomUUID } from "node:crypto";
+export const MAX_SCREENSHOTS_PER_REPORT = 1000;
+export const MAX_REPORT_OBJECT_BYTES = 10 * 1024 * 1024;
+function warn(session, message) {
+    session.failed = true;
+    session.onWarning?.(message);
+}
+function getHttpStatus(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return /\((\d{3})\)/.exec(message)?.[1] ?? null;
+}
+function warnAboutUploadFailure(session) {
+    if (!session.uploadFailed) {
+        return;
+    }
+    const status = session.uploadFailureStatus ? ` (HTTP ${session.uploadFailureStatus})` : "";
+    session.onWarning?.(`Cloud report upload failed${status}; further uploads skipped.`);
+}
+function decodeScreenshot(dataUrl) {
+    const match = /^data:image\/png;base64,([a-zA-Z0-9+/=\s]+)$/.exec(dataUrl);
+    return match ? Buffer.from(match[1], "base64") : null;
+}
+async function uploadObject(uploadPolicy, body, name, contentType, fetchImpl) {
+    const form = new FormData();
+    for (const [key, value] of Object.entries(uploadPolicy.data ?? {})) {
+        form.append(key, String(value));
+    }
+    form.append("file", new Blob([body], { type: contentType }), name);
+    const response = await fetchImpl(uploadPolicy.url, { method: "POST", body: form });
+    if (!response.ok) {
+        throw new Error(`S3 upload failed (${response.status}): ${await response.text()}`);
+    }
+}
+export class DroidReportUploadManager {
+    constructor({ workers = 4, fetchImpl = fetch } = {}) {
+        this.workers = workers;
+        this.fetchImpl = fetchImpl;
+        this.queue = [];
+        this.activeWorkers = 0;
+        this.sessions = new Set();
+    }
+    /**
+     * @param {{
+     *   loadmillClient?: {
+     *     requestDroidRunReportUploadPolicy: (uploadId: string, payload: { name: string, size: number }) => Promise<unknown>,
+     *     associateDroidRunReport: (runId: string, payload: object) => Promise<unknown>,
+     *   },
+     *   onWarning?: ((message: string) => void) | null,
+     * }} options
+     */
+    startSession({ loadmillClient, onWarning = null } = {}) {
+        const session = {
+            uploadId: randomUUID(),
+            loadmillClient,
+            onWarning,
+            sequence: 0,
+            screenshotCount: 0,
+            deduplication: new Map(),
+            pending: new Set(),
+            failed: false,
+            uploadFailed: false,
+            uploadFailureStatus: null,
+            finishing: false,
+            finishPromise: null,
+        };
+        this.sessions.add(session);
+        return session;
+    }
+    enqueueScreenshot(session, dataUrl) {
+        if (!session || session.finishing || session.uploadFailed || typeof dataUrl !== "string") {
+            return null;
+        }
+        const hash = createHash("sha256").update(dataUrl).digest("hex");
+        const existing = session.deduplication.get(hash);
+        if (existing) {
+            return existing;
+        }
+        const body = decodeScreenshot(dataUrl);
+        if (!body) {
+            warn(session, "Failed to upload Droid report screenshot: unsupported image data.");
+            return null;
+        }
+        if (body.length > MAX_REPORT_OBJECT_BYTES) {
+            warn(session, `Droid report screenshot exceeds ${MAX_REPORT_OBJECT_BYTES} bytes.`);
+            return null;
+        }
+        if (session.screenshotCount >= MAX_SCREENSHOTS_PER_REPORT) {
+            warn(session, `Droid report exceeds ${MAX_SCREENSHOTS_PER_REPORT} screenshots.`);
+            return null;
+        }
+        const name = `screenshots/${String(++session.sequence).padStart(4, "0")}.png`;
+        session.screenshotCount += 1;
+        session.deduplication.set(hash, name);
+        this.#enqueue(session, { name, body, contentType: "image/png" });
+        return name;
+    }
+    finish(session, { savedRunId, html } = {}) {
+        if (!session) {
+            return Promise.resolve({ published: false, reason: "missing_session" });
+        }
+        if (session.finishPromise) {
+            return session.finishPromise;
+        }
+        session.finishing = true;
+        session.finishPromise = this.#finish(session, { savedRunId, html }).catch((error) => {
+            const status = getHttpStatus(error);
+            warn(session, `Failed to publish Droid run report${status ? ` (HTTP ${status})` : ""}.`);
+            return { published: false, reason: "publish_failed" };
+        }).finally(() => {
+            this.sessions.delete(session);
+        });
+        return session.finishPromise;
+    }
+    async #finish(session, { savedRunId, html }) {
+        await this.waitForSession(session);
+        if (session.uploadFailed) {
+            warnAboutUploadFailure(session);
+            return { published: false, reason: "upload_failed" };
+        }
+        if (!savedRunId || typeof html !== "string") {
+            return { published: false, reason: "missing_report" };
+        }
+        const body = Buffer.from(html, "utf8");
+        if (body.length > MAX_REPORT_OBJECT_BYTES) {
+            warn(session, `Droid HTML report exceeds ${MAX_REPORT_OBJECT_BYTES} bytes.`);
+            return { published: false, reason: "too_large" };
+        }
+        const htmlUpload = await this.#enqueue(session, {
+            name: "report.html",
+            body,
+            contentType: "text/html; charset=utf-8",
+        });
+        if (session.failed) {
+            warnAboutUploadFailure(session);
+            return { published: false, reason: "upload_failed" };
+        }
+        const key = htmlUpload.key;
+        await session.loadmillClient.associateDroidRunReport(savedRunId, {
+            key,
+            screenshotCount: session.screenshotCount,
+        });
+        return { published: true, key };
+    }
+    async waitForSession(session) {
+        while (session?.pending?.size) {
+            await Promise.all([...session.pending]);
+        }
+        return { failed: Boolean(session?.failed) };
+    }
+    getPendingCount(session) {
+        return session?.pending?.size ?? 0;
+    }
+    /**
+     * @param {{ timeoutMs?: number | null }} options
+     */
+    async drain({ timeoutMs = null } = {}) {
+        const pending = Promise.all([...this.sessions].map((session) => session.finishPromise ?? this.waitForSession(session)));
+        if (!Number.isFinite(timeoutMs)) {
+            await pending;
+            return { timedOut: false };
+        }
+        let timer;
+        const result = await Promise.race([
+            pending.then(() => false),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(true), timeoutMs);
+            }),
+        ]);
+        clearTimeout(timer);
+        return { timedOut: result };
+    }
+    #enqueue(session, object) {
+        let resolveTask;
+        const promise = new Promise((resolve) => {
+            resolveTask = resolve;
+        });
+        session.pending.add(promise);
+        this.queue.push({ session, object, resolveTask, promise });
+        this.#pump();
+        return promise;
+    }
+    #pump() {
+        while (this.activeWorkers < this.workers && this.queue.length > 0) {
+            const task = this.queue.shift();
+            this.activeWorkers += 1;
+            this.#runTask(task).finally(() => {
+                this.activeWorkers -= 1;
+                this.#pump();
+            });
+        }
+    }
+    async #runTask(task) {
+        let result = { uploaded: false, key: null };
+        try {
+            if (task.session.uploadFailed) {
+                return;
+            }
+            const policyResult = await task.session.loadmillClient.requestDroidRunReportUploadPolicy(task.session.uploadId, { name: task.object.name, size: task.object.body.length });
+            const uploadPolicy = policyResult?.uploadPolicy;
+            if (!uploadPolicy?.key) {
+                throw new Error("Loadmill did not return an upload policy.");
+            }
+            await uploadObject(uploadPolicy, task.object.body, task.object.name.split("/").at(-1), task.object.contentType, this.fetchImpl);
+            result = { uploaded: true, key: uploadPolicy.key };
+        }
+        catch (error) {
+            task.session.failed = true;
+            task.session.uploadFailed = true;
+            task.session.uploadFailureStatus ??= getHttpStatus(error);
+        }
+        finally {
+            task.session.pending.delete(task.promise);
+            task.resolveTask(result);
+        }
+    }
+}
+export const droidReportUploadManager = new DroidReportUploadManager();
+export function droidReportScreenshotUrl(savedRunId, objectName) {
+    const fileName = objectName.split("/").at(-1);
+    return `/api/droid-cua/runs/${encodeURIComponent(savedRunId)}/report/screenshots/${encodeURIComponent(fileName)}`;
+}

package/build/src/integrations/loadmill/run-reporter.js

@@ -100,8 +100,12 @@ export async function reportDroidRunCompletion({ loadmillClient, onWarning = nul
         return { reported: false, reason: "missing_run_id" };
     }
     try {
-        await loadmillClient.reportDroidRun(payload);
-        return { reported: true };
+        const response = await loadmillClient.reportDroidRun(payload);
+        return {
+            reported: true,
+            savedRunId: response?.run?.id ?? null,
+            run: response?.run ?? null,
+        };
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);

package/build/src/modes/execution-mode.js

@@ -592,7 +592,8 @@ export class ExecutionMode {
             // clearMessages() restores the base system prompt, but we need to add context
             const recoverySystemPrompt = buildExecutionRecoveryPrompt({
                 basePrompt: this.initialSystemText,
-                transcript: transcriptContext
+                transcript: transcriptContext,
+                explainMode: context?.runtimeConfig?.explainMode === true,
             });
             // Replace the system message with the enhanced one
             this.session.installRecoverySystemPrompt(recoverySystemPrompt);

package/build/src/prompts/design.js

@@ -2,6 +2,7 @@ import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./
 import { buildAppContextSection, buildSecretKeysContextSection } from "./execution.js";
 export function buildDesignModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
+    const explainMode = options.explainMode === true;
     const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
     const appContextSection = buildAppContextSection(appContextText);
     const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
@@ -94,7 +95,14 @@ WRONG Example (DON'T DO THIS):
 3. etc...
 \`\`\`
 
-Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
+${explainMode ? `EXPLAIN MODE:
+- Briefly explain non-obvious decisions, including why you choose one path, skip an expected action, or consider the goal complete
+- Name only the visible evidence that matters to the decision
+- Do not narrate routine clicks, typing, scrolling, or other obvious actions
+- Before a non-obvious action, briefly explain the decision in the same response as that action
+- Include the evidence that proves the requested flow is complete in the same response as the \`test_design_complete\` call
+
+` : ""}Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.${appContextSection ? `\n\n${appContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}
 `;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },

package/build/src/prompts/execution.js

@@ -38,6 +38,7 @@ ${normalized.map((key) => `- ${key}`).join("\n")}`;
 }
 export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
+    const explainMode = options.explainMode === true;
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
     const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
     const appContextSection = buildAppContextSection(appContextText);
@@ -54,12 +55,12 @@ EXECUTION MODE - Critical Behavior:
 You are executing test script commands one at a time. This is an automated test run, not a chat with the user.
 
 CRITICAL RULES:
-- DO NOT generate conversational text or narration
+- ${explainMode ? "Explain non-obvious decisions briefly, including why you skip an expected action or consider the current instruction complete" : "DO NOT generate conversational text or narration"}
 - DO NOT ask questions like "What should I do next?", "Would you like...", "Can I assist...?"
-- DO NOT describe what you see on screen
+- ${explainMode ? "When a decision depends on visible state, name only the evidence that matters to that decision" : "DO NOT describe what you see on screen"}
 - DO NOT say "Let me know if you need help" or similar phrases
-- Just execute the action silently and stop immediately
-- Only generate text if the action FAILED or cannot be completed
+- ${explainMode ? "Do not narrate routine clicks, typing, scrolling, or other obvious actions" : "Just execute the action silently and stop immediately"}
+- ${explainMode ? "Before a non-obvious action, briefly explain the decision in the same response as that action\n- Explain completion evidence using assistant text; do not call `assert` merely to confirm that an ordinary action succeeded\n- Call `assert` only when the current instruction explicitly contains assertion or verification intent\n- If an action FAILED or the instruction cannot be completed, state that clearly and explain why" : "Only generate text if the action FAILED or cannot be completed"}
 - ${browserTarget ? "Use browser-appropriate keyboard input only when it is the most reliable path" : "Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses"}
 - Never repeat the same mutating action with the same apparent intent unless the UI clearly shows failure or no state change
 - If a submit/create/approve/reject/login action appears to succeed, stop instead of trying to reconfirm by doing it again
@@ -86,7 +87,7 @@ ${browserTarget ? "" : `   - Every \`type\` action must end its computer-action
 4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
 5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
 6. Stop as soon as success is visible
-7. Stop immediately - no commentary, no questions
+7. ${explainMode ? "If the completion decision is not obvious, briefly state the visible evidence, then stop - no questions" : "Stop immediately - no commentary, no questions"}
 
 ${instructionContextRule}
 ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
@@ -95,7 +96,7 @@ ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ?
         { title: "Execution Mode Instructions", text: executionCustomText }
     ]);
 }
-export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
+export function buildExecutionRecoveryPrompt({ basePrompt, transcript, explainMode = false }) {
     if (!transcript) {
         return basePrompt;
     }
@@ -104,7 +105,9 @@ export function buildExecutionRecoveryPrompt({ basePrompt, transcript }) {
 [SESSION RECOVERY - Connection was lost. Previous actions completed before the error:]
 ${transcript}
 
-[IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]`;
+${explainMode
+        ? "[IMPORTANT: Resume execution. Continue using concise decision explanations without narrating routine actions.]"
+        : "[IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]"}`;
 }
 export function buildExecutionNavigationRecoveryPrompt({ basePrompt, transcript, navigatedUrl }) {
     const normalizedTranscript = typeof transcript === "string" ? transcript.trim() : "";

package/build/src/utils/console-output.js

@@ -27,6 +27,9 @@ export function formatCliOutput(item) {
         }
         const stepCompletion = stepCompletionData(text);
         if (stepCompletion) {
+            if (item.payload?.explainMode === true) {
+                return `assistant: Step complete — ${stepCompletion.instruction}\nassistant: Reason: ${stepCompletion.reason}`;
+            }
             return `assistant: Step complete — ${stepCompletion.instruction}`;
         }
         return text

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.25.0",
+  "version": "2.27.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",

package/skill/droid-cua/references/running-and-healing.md

@@ -2,17 +2,17 @@
 
 ## Construct the run
 
-Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider. Use Loadmill Smart as the recommended default, or Loadmill Beacon when the user prioritizes faster, lower-cost execution:
+Prefer a single test before a directory or scenario run. Preserve an existing provider/model choice in `ci/droid-cua.json`; otherwise use the Loadmill provider. Use Loadmill Smart as the recommended default, or Loadmill Pulse when the user prioritizes faster, lower-cost execution:
 
 ```sh
 # Recommended default
 --llm-provider loadmill --cua-model loadmill-smart
 
 # Faster, lower-cost option
---llm-provider loadmill --cua-model loadmill-beacon
+--llm-provider loadmill --cua-model loadmill-pulse
 ```
 
-Both Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart and Beacon silently; preserve the project's model or explain the speed/cost tradeoff and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
+All Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart, Pulse, and Beacon silently; preserve the project's model or explain the capability, speed, and experimental tradeoffs and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
 
 Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` for every agent-observed run: it writes the screenshots needed for live progress updates as well as diagnosis.