@loadmill/droid-cua 2.24.0 → 2.25.0

package/README.md

@@ -136,6 +136,11 @@ npm install -g @loadmill/droid-cua
 
 Install the bundled Agent Skill for your coding agent:
 
+In the desktop app, choose **Help → Install Droid CUA Skill**, then select your
+coding agent from the submenu. The app installs the skill globally for your user account.
+
+Alternatively, install it from the CLI:
+
 ```sh
 # Global installation (default)
 droid-cua skill install cursor

package/build/src/cli/headless-runner.js

@@ -86,6 +86,7 @@ function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionP
         toolCapabilities: {
             assert: true,
             navigate: isBrowserTarget,
+            closeKeyboard: !isBrowserTarget,
             loadmill: true,
         },
         secretsByKey: secrets?.secretsByKey || {},
@@ -174,6 +175,7 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
             aimAssistMode: headlessConfig.aimAssistMode,
             testDataContext,
             availableSecretKeys: secrets?.availableSecretKeys || [],
+            cuaModel: runtimeContext.config.cuaModel,
         });
         session.setSystemPrompt(executionPrompt);
         const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({

package/build/src/commands/run.js

@@ -67,8 +67,7 @@ export async function handleRun(args, session, context) {
     if (context.setAgentWorking) {
         context.setAgentWorking(true, 'Executing test...');
     }
-    // Reset session state to prevent context leak from design mode
-    // Each test instruction should execute in isolation
+    // Reset session state to prevent context leak from design mode.
     session.updateResponseId(undefined);
     session.clearMessages();
     const appContext = await resolveExecutionAppContext({ contextPath: context.appContextPath });
@@ -89,7 +88,10 @@ export async function handleRun(args, session, context) {
     }
     // Set execution mode system prompt (replaces any design mode prompt)
     const availableSecretKeys = context.engine.getAvailableSecretKeys();
-    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, { availableSecretKeys });
+    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, {
+        availableSecretKeys,
+        cuaModel: context.runtimeConfig?.cuaModel,
+    });
     session.setSystemPrompt(executionPrompt);
     // Create execution mode
     const executionMode = new ExecutionMode(session, context.engine, instructions);

package/build/src/core/cua-models.js

@@ -1,6 +1,10 @@
 export const DEFAULT_OPENAI_CUA_MODEL = "gpt-5.4";
 export const DEFAULT_LOADMILL_CUA_MODEL = "loadmill-smart";
 export const DEFAULT_CUA_MODEL = DEFAULT_LOADMILL_CUA_MODEL;
+export const CUA_CONVERSATION_POLICIES = Object.freeze({
+    CHAINED: "chained",
+    ISOLATED: "isolated",
+});
 export const CUA_MODEL_OPTIONS = Object.freeze([
     Object.freeze({
         id: "loadmill-smart",
@@ -70,3 +74,8 @@ export function resolveDefaultCuaModelForProvider(providerMode) {
         ? DEFAULT_LOADMILL_CUA_MODEL
         : DEFAULT_OPENAI_CUA_MODEL;
 }
+export function resolveConversationPolicy(cuaModel) {
+    return cuaModel === "loadmill-beacon" || cuaModel === "beacon"
+        ? CUA_CONVERSATION_POLICIES.ISOLATED
+        : CUA_CONVERSATION_POLICIES.CHAINED;
+}

package/build/src/core/execution-engine.js

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
 import { closeCurrentTab as closeCurrentBrowserTab, getBrowserInfo, navigateToUrl as navigateBrowserToUrl } from "../browser/connection.js";
-import { handleModelAction } from "../device/actions.js";
+import { closeKeyboard, handleModelAction } from "../device/actions.js";
 import { sendCUARequest } from "../device/openai.js";
 import { executeLoadmillInstruction } from "../device/loadmill.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
@@ -14,6 +14,7 @@ import { ScreenshotStore } from "./screenshot-store.js";
 import { handlePreviewClick } from "./tool-handlers/preview-click.js";
 import { handleNavigateToolCall } from "./tool-handlers/navigate.js";
 import { handleCloseCurrentTabToolCall } from "./tool-handlers/close-current-tab.js";
+import { handleCloseKeyboardToolCall } from "./tool-handlers/close-keyboard.js";
 import { handleAssertionToolCall } from "./tool-handlers/assert.js";
 import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
 import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
@@ -176,6 +177,8 @@ function formatActionForRecovery(action) {
         }
         case "scroll":
             return "Scrolled";
+        case "close_keyboard":
+            return "Closed the keyboard";
         default:
             return null;
     }
@@ -240,6 +243,7 @@ export class ExecutionEngine {
         this.stepDelayMs = options.stepDelayMs ?? getConfiguredStepDelayMs();
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
+        this.closeKeyboard = options.closeKeyboard || closeKeyboard;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
         this.reviseTestScript = options.reviseTestScript || null;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
@@ -323,6 +327,8 @@ export class ExecutionEngine {
                 return await handleNavigateToolCall(handlerArgs);
             case "close_current_tab":
                 return await handleCloseCurrentTabToolCall(handlerArgs);
+            case "close_keyboard":
+                return await handleCloseKeyboardToolCall(handlerArgs);
             case "loadmill":
                 return await handleLoadmillToolCall(handlerArgs);
             case "type_sensitive_text":

package/build/src/core/session.js

@@ -45,7 +45,7 @@ export class Session {
         this.messages = [{ role: "system", content: prompt }];
     }
     /**
-     * Add a line to the transcript (for error recovery and logging)
+     * Add a line to the scenario transcript for assertions, recovery, and logging.
      */
     addToTranscript(line) {
         this.transcript.push(line);
@@ -57,7 +57,7 @@ export class Session {
         return this.transcript.join("\n");
     }
     /**
-     * Clear the per-instruction transcript.
+     * Clear the scenario transcript at a run boundary.
      */
     clearTranscript() {
         this.transcript = [];

package/build/src/core/tool-handlers/close-keyboard.js

@@ -0,0 +1,103 @@
+import { CAPTURE_SOURCES } from "../capture-sources.js";
+import { KEYBOARD_CLOSE_RESULTS, buildKeyboardCloseResult } from "../../device/keyboard.js";
+import { printCliOutput } from "../../utils/console-output.js";
+import { emitDesktopDebug } from "../../utils/desktop-debug.js";
+import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, isDesignContext, parseJsonArguments, } from "./utils.js";
+function parseDismissalPoint(argumentsText, deviceInfo = {}) {
+    const parsed = parseJsonArguments(argumentsText || "{}");
+    if (!parsed.ok || !parsed.value || typeof parsed.value !== "object" || Array.isArray(parsed.value)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: expected a JSON object." };
+    }
+    const hasX = Object.prototype.hasOwnProperty.call(parsed.value, "x");
+    const hasY = Object.prototype.hasOwnProperty.call(parsed.value, "y");
+    if (!hasX && !hasY) {
+        return { ok: true, value: null };
+    }
+    if (!hasX || !hasY || !Number.isFinite(parsed.value.x) || !Number.isFinite(parsed.value.y)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: x and y must both be finite numbers." };
+    }
+    const point = { x: parsed.value.x, y: parsed.value.y };
+    const width = Number(deviceInfo.scaled_width);
+    const height = Number(deviceInfo.scaled_height);
+    // We can validate coordinate shape and bounds, but only the model can judge
+    // whether a cloud-iOS point is a safe dismissal target in the current UI.
+    if (point.x < 0 || point.y < 0 ||
+        (Number.isFinite(width) && point.x >= width) ||
+        (Number.isFinite(height) && point.y >= height)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: the dismissal point is outside the current screenshot." };
+    }
+    return { ok: true, value: point };
+}
+export async function handleCloseKeyboardToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    if (session?.targetKind === "browser" || session?.deviceInfo?.platform === "browser") {
+        return {
+            kind: "continue",
+            outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: "close_keyboard is only supported for mobile runs." })),
+        };
+    }
+    const parsedPoint = parseDismissalPoint(argumentsText, session.deviceInfo);
+    if (!parsedPoint.ok) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, parsedPoint.message),
+        };
+    }
+    const dismissalPoint = parsedPoint.value;
+    let result;
+    try {
+        result = await engine.closeKeyboard(session.deviceId, {
+            dismissalPoint,
+            scale: session.deviceInfo?.scale || 1,
+        });
+    }
+    catch (error) {
+        result = buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+    const addOutput = context?.addOutput || printCliOutput;
+    const outputText = result.status === KEYBOARD_CLOSE_RESULTS.FAILED
+        ? `${result.message}. The preceding text entry was unaffected.`
+        : result.message;
+    const scope = isDesignContext(context) ? "design" : "execution";
+    const ids = {
+        ...(scope === "design" ? { sessionId: context?.sessionId } : { runId: context?.runId }),
+        ...(stepContext?.stepId ? { stepId: stepContext.stepId } : {}),
+        ...(Number.isInteger(stepContext?.instructionIndex)
+            ? { instructionIndex: stepContext.instructionIndex }
+            : {}),
+    };
+    const payload = {
+        platform: session.deviceInfo?.platform,
+        toolName: "close_keyboard",
+        callId,
+        status: result.status,
+        failed: result.status === KEYBOARD_CLOSE_RESULTS.FAILED,
+        ...(dismissalPoint ? { dismissalPoint } : {}),
+        ...(result.error ? { error: result.error } : {}),
+    };
+    emitDesktopDebug("tool_call", scope, ids, payload);
+    addOutput({
+        type: "action",
+        text: result.message,
+        eventType: "tool_call",
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        payload,
+    });
+    session.addToTranscript(`[Close keyboard] ${outputText}`);
+    const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
+    const currentScreenshot = await engine.recordScreenshot(screenshotBase64, {
+        runId: context?.runId,
+        sessionId: context?.sessionId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
+    });
+    return {
+        kind: "continue",
+        outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: outputText, imageBase64: screenshotBase64 })),
+        currentScreenshot,
+        executedActions: [{ type: "close_keyboard", ...(dismissalPoint || {}) }],
+    };
+}

package/build/src/device/actions.js

@@ -20,3 +20,10 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
     const backend = getDeviceBackend(platform);
     return backend.handleModelAction(deviceId, action, scale, context);
 }
+export async function closeKeyboard(deviceId, options = {}) {
+    const platform = getCurrentPlatform();
+    if (!platform) {
+        throw new Error("No platform set. Call connectToDevice first.");
+    }
+    return getDeviceBackend(platform).closeKeyboard(deviceId, options);
+}

package/build/src/device/android/actions.js

@@ -6,9 +6,25 @@ import { buildMobileKeypressPayload, getAndroidAdbTransport, normalizeMobileKeyp
 import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
 import { execAdb } from "./tools.js";
+import { closeAdbKeyboard, resolveAdbKeyboardVisibility } from "../keyboard.js";
 function adbShell(deviceId, command) {
     return execAdb(["-s", deviceId, "shell", command]);
 }
+async function getKeyboardVisibility(deviceId) {
+    const { stdout: inputMethodDump } = await adbShell(deviceId, "dumpsys input_method");
+    const { stdout: showImeWithHardwareKeyboard } = await adbShell(deviceId, "settings get secure show_ime_with_hard_keyboard");
+    return resolveAdbKeyboardVisibility(inputMethodDump, showImeWithHardwareKeyboard);
+}
+export async function closeKeyboard(deviceId) {
+    // KEYCODE_BACK dismisses the Android IME, but it would navigate if the app
+    // already hid the keyboard. getKeyboardVisibility also filters the hardware
+    // keyboard configuration where mInputShown is true without a software input
+    // view. closeAdbKeyboard invokes Back only for an affirmative result.
+    return closeAdbKeyboard({
+        getVisibility: () => getKeyboardVisibility(deviceId),
+        pressBack: () => adbShell(deviceId, "input keyevent KEYCODE_BACK"),
+    });
+}
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const meta = (payload = {}) => buildActionEventMeta({

package/build/src/device/cloud/actions.js

@@ -6,6 +6,33 @@ import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSe
 import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
+import { closeAppiumKeyboard, closeAppiumKeyboardWithTap } from "../keyboard.js";
+export async function closeKeyboard(_deviceId, { dismissalPoint = null, scale = 1 } = {}) {
+    const session = getActiveSession();
+    if (!session?.client) {
+        throw new Error("No active cloud session");
+    }
+    if (session.platform === "ios") {
+        // XCUITest hideKeyboard was unreliable on the tested AWS real devices, and
+        // Return/newline can submit forms. Trust the model to identify a dismissal
+        // control or inert background, but tap only after confirming the keyboard
+        // is open. With no trusted point, fail safely without touching the screen.
+        const dpr = getDevicePixelRatio();
+        const tap = dismissalPoint
+            ? () => session.client.tap(session.sessionId, Math.round((dismissalPoint.x / scale) / dpr), Math.round((dismissalPoint.y / scale) / dpr))
+            : null;
+        return closeAppiumKeyboardWithTap({
+            isKeyboardShown: () => session.client.isKeyboardShown(session.sessionId),
+            tap,
+        });
+    }
+    // UiAutomator2's native query/hide commands are reliable on cloud Android and
+    // avoid the navigation risk of treating Android Back as a generic fallback.
+    return closeAppiumKeyboard({
+        isKeyboardShown: () => session.client.isKeyboardShown(session.sessionId),
+        hideKeyboard: () => session.client.hideKeyboard(session.sessionId),
+    });
+}
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const session = getActiveSession();

package/build/src/device/cloud/cloud-appium-client.js

@@ -99,6 +99,21 @@ export class CloudAppiumClient {
             ]
         });
     }
+    async isKeyboardShown(sessionId) {
+        const response = await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: isKeyboardShown",
+            args: [{}]
+        });
+        return response.value === true;
+    }
+    async hideKeyboard(sessionId) {
+        // Currently called by cloud Android only. Cloud iOS deliberately routes to
+        // its guarded model-selected tap in cloud/actions.js.
+        await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: hideKeyboard",
+            args: [{}]
+        });
+    }
     async scroll(sessionId, startX, startY, endX, endY, duration = 500) {
         await this.request("POST", `/session/${sessionId}/actions`, {
             actions: [

package/build/src/device/connection.js

@@ -4,7 +4,7 @@
  * Thin wrapper that delegates to the appropriate platform backend.
  * Maintains backwards compatibility with existing code.
  */
-import { getDeviceBackend, detectPlatform, setCurrentPlatform, setCurrentSource, getCurrentPlatform } from "./factory.js";
+import { getDeviceBackend, detectPlatform, setCurrentPlatform, setCurrentSource, getCurrentPlatform, getCurrentSource } from "./factory.js";
 let currentBackend = null;
 /**
  * Connect to a device (Android target ID/serial or iOS simulator name)
@@ -40,7 +40,10 @@ export async function getDeviceInfo(deviceId, options = {}) {
     if (!currentBackend) {
         throw new Error("Not connected to a device. Call connectToDevice first.");
     }
-    return currentBackend.getDeviceInfo(deviceId, options);
+    return {
+        ...await currentBackend.getDeviceInfo(deviceId, options),
+        source: getCurrentSource(),
+    };
 }
 /**
  * Get screenshot as base64 string

package/build/src/device/factory.js

@@ -47,6 +47,7 @@ export function getDeviceBackend(platform) {
             getDeviceInfo: cloudConnection.getDeviceInfo,
             getScreenshotAsBase64: cloudConnection.getScreenshotAsBase64,
             handleModelAction: cloudActions.handleModelAction,
+            closeKeyboard: cloudActions.closeKeyboard,
             disconnect: cloudConnection.disconnect,
         };
     }
@@ -56,6 +57,7 @@ export function getDeviceBackend(platform) {
             getDeviceInfo: iosConnection.getDeviceInfo,
             getScreenshotAsBase64: iosConnection.getScreenshotAsBase64,
             handleModelAction: iosActions.handleModelAction,
+            closeKeyboard: iosActions.closeKeyboard,
             disconnect: iosConnection.disconnect,
         };
     }
@@ -65,6 +67,7 @@ export function getDeviceBackend(platform) {
         getDeviceInfo: androidConnection.getDeviceInfo,
         getScreenshotAsBase64: androidConnection.getScreenshotAsBase64,
         handleModelAction: androidActions.handleModelAction,
+        closeKeyboard: androidActions.closeKeyboard,
         disconnect: async () => { }, // Android doesn't need explicit disconnect
     };
 }

package/build/src/device/interface.js

@@ -14,6 +14,7 @@
  * @property {string} [platform] - Device platform ('android' or 'ios')
  * @property {string} [device_name] - Human-readable device or simulator name
  * @property {string} [model] - Device model identifier/name
+ * @property {string} [source] - Device source such as 'local' or a cloud provider ID
  */
 /**
  * @typedef {Object} ActionContext
@@ -41,6 +42,9 @@
  *   - Executes an action from the CUA model
  *   - Supported action types: click, type, scroll, drag, keypress, wait
  *   - keypress is limited to single mobile-safe keys only; desktop shortcut chords are unsupported
+ *
+ * closeKeyboard(deviceId: string, options?: {dismissalPoint?: {x: number, y: number}, scale?: number}): Promise<{status: string, message: string, error?: string}>
+ *   - Safely attempts to close the software keyboard without changing app navigation
  */
 export const SUPPORTED_ACTIONS = [
     'click', // Tap at (x, y) coordinates

package/build/src/device/ios/actions.js

@@ -12,6 +12,19 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
+import { closeAppiumKeyboard } from "../keyboard.js";
+export async function closeKeyboard() {
+    const session = getActiveSession();
+    if (!session) {
+        throw new Error("No active iOS session");
+    }
+    // The local simulator supports Appium's keyboard command reliably, so it
+    // needs neither a model-selected tap nor a direct Return keypress.
+    return closeAppiumKeyboard({
+        isKeyboardShown: () => appium.isKeyboardShown(session.sessionId),
+        hideKeyboard: () => appium.hideKeyboard(session.sessionId),
+    });
+}
 /**
  * Handle an action from the CUA model
  * @param {string} simulatorId - The simulator UDID

package/build/src/device/ios/appium-client.js

@@ -123,6 +123,22 @@ export async function type(sessionId, text) {
         ],
     });
 }
+export async function isKeyboardShown(sessionId) {
+    const response = await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: isKeyboardShown",
+        args: [{}],
+    });
+    return response.value === true;
+}
+export async function hideKeyboard(sessionId) {
+    // This XCUITest form is used only by the locally validated simulator path.
+    // Real cloud iOS devices use a guarded tap instead because this command was
+    // unreliable there and sending Return directly can submit the current form.
+    await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: hideKeyboard",
+        args: [{ keys: ["return"] }],
+    });
+}
 /**
  * Perform a scroll action using W3C Actions
  * @param {string} sessionId

package/build/src/device/keyboard.js

@@ -0,0 +1,119 @@
+import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+/**
+ * Keyboard dismissal is intentionally platform-specific:
+ *
+ * - Local Android uses ADB. Back is the reliable dismissal key, but it can also
+ *   navigate away. Android's mInputShown flag stays true when a hardware
+ *   keyboard suppresses the software keyboard, so that configuration is
+ *   treated as already closed before Back is considered.
+ * - Local iOS and cloud Android use Appium's native keyboard commands, which
+ *   worked reliably in device testing and do not require choosing a tap target.
+ * - Cloud iOS uses a model-selected tap. XCUITest's hideKeyboard command was
+ *   unreliable on the tested real devices, while Return/newline could submit a
+ *   form. The caller therefore supplies a trusted dismissal point instead.
+ *
+ * Every strategy follows the same query -> dismiss -> wait -> verify shape and
+ * returns the normalized result below.
+ */
+export const KEYBOARD_CLOSE_RESULTS = {
+    CLOSED: "closed",
+    ALREADY_CLOSED: "already_closed",
+    FAILED: "failed",
+};
+const KEYBOARD_CLOSE_MESSAGES = {
+    [KEYBOARD_CLOSE_RESULTS.CLOSED]: "Keyboard closed",
+    [KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED]: "Keyboard closed",
+    [KEYBOARD_CLOSE_RESULTS.FAILED]: "Keyboard could not be closed and may still be open",
+};
+export function buildKeyboardCloseResult(status, error = null) {
+    return {
+        status,
+        message: KEYBOARD_CLOSE_MESSAGES[status],
+        ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}),
+    };
+}
+export function parseAdbKeyboardVisibility(output) {
+    const matches = [...String(output ?? "").matchAll(/\bmInputShown=(true|false)\b/g)]
+        .map((match) => match[1] === "true");
+    const states = new Set(matches);
+    // Multiple contradictory sections can appear while the IME is transitioning.
+    // Treat missing or ambiguous state as unknown so callers never risk Back.
+    return states.size === 1 ? matches[0] : null;
+}
+export function parseAdbHardwareKeyboardVisibility(output) {
+    const matches = [...String(output ?? "").matchAll(/\b(?:qwerty|12key)\/[vh]\/(v|h)\b/g)]
+        .map((match) => match[1] === "v");
+    const states = new Set(matches);
+    return states.size === 1 ? matches[0] : null;
+}
+export function resolveAdbKeyboardVisibility(inputMethodDump, showImeWithHardwareKeyboard) {
+    const softwareKeyboardRequested = parseAdbKeyboardVisibility(inputMethodDump);
+    const hardwareKeyboardVisible = parseAdbHardwareKeyboardVisibility(inputMethodDump);
+    const showSoftwareKeyboard = String(showImeWithHardwareKeyboard ?? "").trim();
+    // mInputShown means Android asked the IME to show; it does not guarantee that
+    // the input view appeared. With an exposed hardware keyboard and this setting
+    // disabled, only an IME accessory toolbar may be visible, and Back would be
+    // delivered to the app instead of dismissing a software keyboard.
+    if (showSoftwareKeyboard === "0" && hardwareKeyboardVisible === true) {
+        return false;
+    }
+    return softwareKeyboardRequested;
+}
+async function waitForKeyboardUpdate() {
+    const delayMs = getConfiguredStepDelayMs();
+    if (delayMs > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+}
+export async function closeAppiumKeyboard({ isKeyboardShown, hideKeyboard }) {
+    try {
+        if (!await isKeyboardShown()) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED);
+        }
+        await hideKeyboard();
+        await waitForKeyboardUpdate();
+        return await isKeyboardShown()
+            ? buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED)
+            : buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.CLOSED);
+    }
+    catch (error) {
+        return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+}
+export async function closeAppiumKeyboardWithTap({ isKeyboardShown, tap }) {
+    try {
+        if (!await isKeyboardShown()) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED);
+        }
+        if (typeof tap !== "function") {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED);
+        }
+        await tap();
+        await waitForKeyboardUpdate();
+        return await isKeyboardShown()
+            ? buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED)
+            : buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.CLOSED);
+    }
+    catch (error) {
+        return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+}
+export async function closeAdbKeyboard({ getVisibility, pressBack }) {
+    try {
+        const visibility = await getVisibility();
+        if (visibility === false) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED);
+        }
+        if (visibility !== true) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED);
+        }
+        await pressBack();
+        await waitForKeyboardUpdate();
+        return await getVisibility() === false
+            ? buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.CLOSED)
+            : buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED);
+    }
+    catch (error) {
+        return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+}

package/build/src/device/openai.js

@@ -99,6 +99,21 @@ const closeCurrentTabTool = {
         additionalProperties: false,
     },
 };
+const closeKeyboardTool = {
+    type: "function",
+    name: "close_keyboard",
+    description: "Close the mobile software keyboard after text entry. Call this when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden. Do not call it for a slim hardware-keyboard toolbar or suggestions alone. On cloud iOS, optionally provide x and y for a dismissal control or clearly noninteractive background that you trust; never target Return, Go, Search, Next, Continue, Send, or an app action.",
+    strict: false,
+    parameters: {
+        type: "object",
+        properties: {
+            x: { type: "number", description: "Optional horizontal coordinate in the current screenshot." },
+            y: { type: "number", description: "Optional vertical coordinate in the current screenshot." },
+        },
+        required: [],
+        additionalProperties: false,
+    },
+};
 const loadmillTool = {
     type: "function",
     name: "loadmill",
@@ -116,7 +131,7 @@ const loadmillTool = {
 const typeSensitiveTextTool = {
     type: "function",
     name: "type_sensitive_text",
-    description: "Type a secret value by key without exposing the value to the model. Use this only for secret keys listed in the prompt.",
+    description: "Type a secret value by key without exposing the value to the model. Use this only for secret keys listed in the prompt. On mobile, apply the prompt's keyboard-dismissal rules afterward.",
     strict: true,
     parameters: {
         type: "object",
@@ -175,6 +190,9 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
     if (toolCapabilities.closeCurrentTab) {
         tools.push(closeCurrentTabTool);
     }
+    if (toolCapabilities.closeKeyboard) {
+        tools.push(closeKeyboardTool);
+    }
     if (toolCapabilities.loadmill) {
         tools.push(loadmillTool);
     }

package/build/src/modes/design-mode-ink.js

@@ -41,6 +41,7 @@ export class DesignModeInk {
             assert: true,
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
+            closeKeyboard: this.session?.deviceInfo?.platform !== "browser",
             loadmill: false,
             typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,

package/build/src/modes/design-mode.js

@@ -38,6 +38,7 @@ export class DesignMode {
             assert: true,
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
+            closeKeyboard: this.session?.deviceInfo?.platform !== "browser",
             loadmill: false,
             typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,

package/build/src/modes/execution-mode.js

@@ -1,4 +1,5 @@
 import { CAPTURE_SOURCES } from "../core/capture-sources.js";
+import { CUA_CONVERSATION_POLICIES, resolveConversationPolicy } from "../core/cua-models.js";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
 import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, } from "../browser/connection.js";
 import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
@@ -22,7 +23,8 @@ const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
 ]);
 const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
 const LLM_REQUEST_RATE_LIMIT_ERROR_CODE = "llm_rate_limit_exceeded";
-const DEFAULT_ASSERTION_CONTEXT = "No additional prior context is needed; validate directly from the current screen.";
+const OPENAI_INSUFFICIENT_QUOTA_ERROR_CODE = "insufficient_quota";
+const DEFAULT_ASSERTION_CONTEXT = "Use the session transcript to resolve references to prior scenario steps.";
 function isLlmUsageLimitError(err) {
     if (!err || typeof err !== "object") {
         return false;
@@ -35,15 +37,31 @@ function isLlmUsageLimitError(err) {
 function isLlmRequestRateLimitError(err) {
     return err && typeof err === "object" && err.code === LLM_REQUEST_RATE_LIMIT_ERROR_CODE;
 }
+function isOpenAiInsufficientQuotaError(err) {
+    return err && typeof err === "object" && err.code === OPENAI_INSUFFICIENT_QUOTA_ERROR_CODE;
+}
+function isRejectedContinuationError(err) {
+    if (!err || typeof err !== "object") {
+        return false;
+    }
+    return err.param === "previous_response_id" || [
+        "provider_continuation_mismatch",
+        "unknown_previous_response_id",
+        "invalid_previous_response_id",
+    ].includes(err.code);
+}
 function getLlmUsageLimitMessage() {
     return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
 }
 function getLlmRequestRateLimitMessage() {
     return "Execution stopped: too many AI requests are running for this account right now. Please retry shortly or reduce parallel test runs.";
 }
+function getOpenAiInsufficientQuotaMessage() {
+    return "Execution stopped: your OpenAI account has insufficient quota. Check your OpenAI plan and billing details.";
+}
 /**
  * Execution Mode - Run test scripts line-by-line
- * Each instruction is executed in isolation (messages cleared after each turn)
+ * Instructions share scenario context unless the selected model requires isolation.
  */
 export class ExecutionMode {
     constructor(session, executionEngine, instructions, isHeadlessMode = false) {
@@ -77,6 +95,7 @@ export class ExecutionMode {
             assert: true,
             navigate: this.isBrowserSession,
             closeCurrentTab: this.isBrowserSession,
+            closeKeyboard: !this.isBrowserSession,
             loadmill: true,
             typeSensitiveText: this.engine.hasSecrets(),
         };
@@ -132,8 +151,9 @@ export class ExecutionMode {
             payload: extra.payload
         });
     }
-    resetInstructionConversationState() {
+    resetScenarioConversationState() {
         this.session.updateResponseId(null);
+        this.session.clearMessages();
         if (typeof this.session.clearTranscript === "function") {
             this.session.clearTranscript();
             return;
@@ -142,17 +162,53 @@ export class ExecutionMode {
             this.session.transcript = [];
         }
     }
-    async executeVerificationInstruction(instruction, context, stepContext = null) {
-        const toolResult = await handleAssertionToolCall({
-            engine: this.engine,
-            callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
-            argumentsText: JSON.stringify({
-                condition: extractAssertionPrompt(instruction),
-                context: DEFAULT_ASSERTION_CONTEXT,
-            }),
-            context,
-            stepContext,
+    startInstructionConversation(context) {
+        const conversationPolicy = resolveConversationPolicy(context?.runtimeConfig?.cuaModel);
+        if (conversationPolicy === CUA_CONVERSATION_POLICIES.ISOLATED) {
+            this.session.updateResponseId(null);
+        }
+    }
+    stopForOpenAiInsufficientQuota(err, context, stepContext) {
+        const addOutput = context.addOutput || printCliOutput;
+        const stopMessage = getOpenAiInsufficientQuotaMessage();
+        logger.debug('Stopping execution because the OpenAI account has insufficient quota', {
+            instruction: context?.currentInstruction,
+            message: err.message,
+            status: err.status,
+            code: err.code,
+            type: err.type,
+        });
+        this.emit(addOutput, 'error', stopMessage, context, stepContext, {
+            eventType: 'error',
+            payload: {
+                message: stopMessage,
+                status: err.status,
+                code: err.code,
+                type: err.type,
+            }
         });
+        return { success: false, error: stopMessage, usageLimitReached: true };
+    }
+    async executeVerificationInstruction(instruction, context, stepContext = null) {
+        let toolResult;
+        try {
+            toolResult = await handleAssertionToolCall({
+                engine: this.engine,
+                callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
+                argumentsText: JSON.stringify({
+                    condition: extractAssertionPrompt(instruction),
+                    context: DEFAULT_ASSERTION_CONTEXT,
+                }),
+                context,
+                stepContext,
+            });
+        }
+        catch (err) {
+            if (isOpenAiInsufficientQuotaError(err)) {
+                return this.stopForOpenAiInsufficientQuota(err, context, stepContext);
+            }
+            throw err;
+        }
         const assertionStatus = toolResult.summaryUpdates?.assertionStatus;
         if (assertionStatus === "passed") {
             this.stats.assertionsPassed++;
@@ -184,6 +240,7 @@ export class ExecutionMode {
         const addOutput = runContext.addOutput || printCliOutput;
         // Start timing
         this.stats.startTime = Date.now();
+        this.resetScenarioConversationState();
         for (let i = 0; i < this.instructions.length; i++) {
             const instruction = this.instructions[i];
             const stepContext = this.buildStepContext(i, instruction);
@@ -288,7 +345,7 @@ export class ExecutionMode {
             futureInstructions: this.instructions.slice((stepContext?.instructionIndex ?? -1) + 1),
         };
         if (retryCount === 0) {
-            this.resetInstructionConversationState();
+            this.startInstructionConversation(context);
         }
         this.session.addToTranscript(`[User] ${instruction}`);
         this.session.addMessage("user", instruction);
@@ -364,6 +421,18 @@ export class ExecutionMode {
                 stack: err.stack
             });
             const addOutput = context.addOutput || printCliOutput;
+            if (this.session.previousResponseId && isRejectedContinuationError(err)) {
+                this.emit(addOutput, 'error', err.message, context, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: err.message,
+                        status: err.status,
+                        code: err.code,
+                        param: err.param,
+                    }
+                });
+                return { success: false, error: err.message };
+            }
             if (isLlmRequestRateLimitError(err)) {
                 const stopMessage = getLlmRequestRateLimitMessage();
                 logger.debug('Stopping execution because an LLM request rate limit was reached', {
@@ -384,6 +453,9 @@ export class ExecutionMode {
                 });
                 return { success: false, error: stopMessage, usageLimitReached: true };
             }
+            if (isOpenAiInsufficientQuotaError(err)) {
+                return this.stopForOpenAiInsufficientQuota(err, context, stepContext);
+            }
             if (isLlmUsageLimitError(err)) {
                 const stopMessage = getLlmUsageLimitMessage();
                 logger.debug('Stopping execution because an LLM usage limit was reached', {
@@ -448,8 +520,8 @@ export class ExecutionMode {
                     }
                     else {
                         const platform = this.getCurrentPlatform();
-                        const deviceName = this.session.deviceName || undefined;
-                        const deviceId = await this.connectToDevice(deviceName, platform);
+                        const deviceTarget = this.session.deviceId || this.session.deviceName || undefined;
+                        const deviceId = await this.connectToDevice(deviceTarget, platform);
                         const deviceInfo = await this.getDeviceInfo(deviceId);
                         this.session.deviceId = deviceId;
                         this.session.deviceInfo = deviceInfo;

package/build/src/prompts/base.js

@@ -30,6 +30,11 @@ ${customSection}
 function isBrowserTarget(deviceInfo = {}) {
     return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
 }
+function isCloudIosTarget(deviceInfo = {}) {
+    const platform = deviceInfo.platform?.trim().toLowerCase();
+    const source = deviceInfo.source?.trim().toLowerCase() || "local";
+    return platform === "ios" && source !== "local";
+}
 function describeControlledTarget(deviceInfo = {}) {
     const platform = typeof deviceInfo.platform === "string" ? deviceInfo.platform.trim().toLowerCase() : "";
     const deviceName = typeof deviceInfo.device_name === "string" ? deviceInfo.device_name.trim() : "";
@@ -38,7 +43,8 @@ function describeControlledTarget(deviceInfo = {}) {
         return browserName ? `a browser session (${browserName})` : "a browser session";
     }
     if (platform === "ios") {
-        return deviceName ? `an iOS simulator (${deviceName})` : "an iOS device";
+        const target = isCloudIosTarget(deviceInfo) ? "an iOS device" : "an iOS simulator";
+        return deviceName ? `${target} (${deviceName})` : target;
     }
     if (platform === "android") {
         return deviceName ? `an Android device (${deviceName})` : "an Android device";
@@ -150,6 +156,12 @@ function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
 }
 function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
+    const cloudIosKeyboardGuidance = isCloudIosTarget(deviceInfo) ? `
+  - On this cloud iOS device, \`close_keyboard\` can optionally tap a point you choose from the current screenshot.
+  - If you can identify a dedicated keyboard dismissal control or clearly noninteractive background that you trust, pass its \`x\` and \`y\` coordinates to \`close_keyboard\`.
+  - Never use Return, Go, Search, Next, Continue, Send, or an app action as the dismissal point because it may submit or mutate the app.
+  - If dismissal is required but no safe point is identifiable, call \`close_keyboard\` without coordinates. Do not guess.
+  - Choose coordinates directly from the post-type screenshot; do not call \`preview_click\` between \`type\` and \`close_keyboard\`.` : "";
     return `
   You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
   Follow the user's instructions to interact with the device.
@@ -164,18 +176,31 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
   - Reason carefully about the approximate pixel position.
   ${buildAimAssistGuidance("mobile", aimAssistMode)}
 
-  Available actions: click, scroll, type, keypress, wait, screenshot.
+  Available actions: click, scroll, type, keypress, wait, screenshot. You also have a \`close_keyboard\` tool.
 
   CRITICAL - Mobile Input Constraints:
   - This is a mobile device, not a desktop. Do NOT use desktop keyboard shortcuts or modifier chords.
   - NEVER emit key combinations such as CTRL+A, CMD+A, CTRL+C, CTRL+V, ALT+TAB, SHIFT+ENTER, or similar shortcuts.
   - Use 'keypress' only for a single mobile-safe key when absolutely necessary.
   - To replace text, tap into the field and type the desired value. If correction is needed, use mobile-safe deletion only.
+  - Every \`type\` action must be the final computer action in its batch.
+  - After \`type\`, inspect its post-action screenshot before deciding whether dismissal is needed.
+  - If a full software keyboard is visibly covering part of the screen, your next action must be \`close_keyboard\`.
+  - Do not finish the current instruction while a required \`close_keyboard\` action is still pending.
+  - A slim hardware-keyboard toolbar, autocomplete suggestions, or a focused field without a full software keyboard does not require \`close_keyboard\`.
+  - Treat \`type_sensitive_text\` as text entry too and apply the same dismissal decision after it returns.
+  - Whenever dismissal is required, do not click, scroll, inspect, submit, type again, issue a keypress, or call another tool before \`close_keyboard\`.
+  - Consecutive DELETE or BACKSPACE keypresses may be used for correction without closing the keyboard between them. After typing the replacement, call \`close_keyboard\`.
+  - Password and other secure fields may appear empty in screenshots even after text was entered successfully.
+  - On iOS, a secure field may also hide an open keyboard from Appium screenshots. After typing into an iOS secure field, call \`close_keyboard\` even when the keyboard is not visible.
+  - Do not type into a secure field again solely because its characters or bullets are not visible.
+  - Retry secure-field input only when there is separate evidence of failure, such as another field changing or the app rejecting the value.
+  ${cloudIosKeyboardGuidance}
   - Prefer tapping visible controls over hardware key events.
   - Prefer on-screen navigation controls such as menus, tabs, drawer items, back arrows, close buttons, and explicit logout buttons over keypress actions.
   - Do NOT use Back or ESC for normal app navigation when a reliable on-screen control is visible.
   - Avoid using Back or ESC from a main or root screen, because it may leave the app.
-  - Exception: if the software keyboard is open and blocking the next needed control, Back or ESC may be used to dismiss the keyboard before continuing.
+  - Never use Back or ESC to dismiss the software keyboard; use \`close_keyboard\`.
   - Treat keypress actions as a fallback for limited cases only, such as a clearly needed single mobile-safe key or dismissing transient UI when no better visible control exists.
 
   CRITICAL - Automatic Timing:
@@ -185,7 +210,7 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
 
   CRITICAL - Mutating Actions:
   - Mutating actions change app state. Examples: submit, create, save, confirm, approve, reject, login, logout, send, place order, initiate transfer
-  - Before tapping a mutating action button, dismiss the software keyboard first when it is open and not required for the tap
+  - Before tapping a mutating action button, call \`close_keyboard\` first when the dismissal rules above require it
   - After performing a mutating action once, do NOT repeat the same mutating action unless the UI clearly shows the first attempt failed or had no effect
   - Treat visible state change as success. Examples: form fields clear, submit button returns to normal, status changes, list refreshes, new row appears, success message appears, screen changes
   - For form submissions specifically, if the relevant fields clear and the action button returns to its normal idle state, treat that as success even if the new row or confirmation is not obvious yet

package/build/src/prompts/execution.js

@@ -1,4 +1,5 @@
 import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
+import { CUA_CONVERSATION_POLICIES, resolveConversationPolicy } from "../core/cua-models.js";
 import { normalizeSecretKeys } from "../core/secrets.js";
 import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
 export function buildAppContextSection(text) {
@@ -43,10 +44,14 @@ export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, ap
     const testDataContextSection = buildTestDataContextSection(options.testDataContext);
     const secretKeysContextSection = buildSecretKeysContextSection(options.availableSecretKeys);
     const browserTarget = isBrowserTarget(deviceInfo);
+    const conversationPolicy = resolveConversationPolicy(options.cuaModel);
+    const instructionContextRule = conversationPolicy === CUA_CONVERSATION_POLICIES.ISOLATED
+        ? "This model starts each instruction without prior model history. Execute only the current instruction and stop when it is complete."
+        : "Instructions are sequential parts of one scenario. Use prior instructions and outcomes to resolve references in the current instruction, but execute only the current instruction. Preserve expected values established earlier; never replace them with the observed state merely to produce a pass.";
     const prompt = `${basePrompt}
 
 EXECUTION MODE - Critical Behavior:
-You are executing test script commands one at a time. This is NOT a conversation.
+You are executing test script commands one at a time. This is an automated test run, not a chat with the user.
 
 CRITICAL RULES:
 - DO NOT generate conversational text or narration
@@ -70,14 +75,20 @@ Your process:
    - Before calling \`assert\`, make sure the current screen contains the evidence needed to validate the condition. If needed, perform only minimal preparation first, such as taking a screenshot, waiting briefly, closing the keyboard, scrolling within the relevant view, or opening the relevant menu.
    - Once you call \`assert\`, validation is read-only. The assertion tool will not click, type, wait, scroll, request screenshots, or change the device state.
    - Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
+${browserTarget ? "" : `   - Every \`type\` action must end its computer-action batch so you can inspect the post-type screenshot
+   - Call \`close_keyboard\` next when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden
+   - Do not complete the current instruction while that required \`close_keyboard\` action is pending
+   - Otherwise, including when only a slim hardware-keyboard toolbar or suggestions are visible, continue without \`close_keyboard\`
+   - Apply the same dismissal decision after \`type_sensitive_text\`
+`}
    - Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
 3. Execute the required actions
-4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
+4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
 5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
 6. Stop as soon as success is visible
 7. Stop immediately - no commentary, no questions
 
-Each instruction is independent. Do not reference previous instructions or ask about next steps.
+${instructionContextRule}
 ${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}${secretKeysContextSection ? `\n\n${secretKeysContextSection}` : ""}`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.24.0",
+  "version": "2.25.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",

package/skill/droid-cua/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: droid-cua
-description: Write, run, monitor, and troubleshoot Droid CUA mobile and web tests. Use when an agent needs to onboard @loadmill/droid-cua, configure Loadmill authentication, discover Android devices or AVDs, prepare iOS simulators or local browsers, author or revise .dcua tests and project context, run tests locally or on supported cloud devices, analyze reports and debug artifacts, distinguish test failures from product bugs, or perform guarded test healing.
+description: Operate Droid CUA `.dcua` test projects. Use when the user explicitly names Droid CUA or @loadmill/droid-cua, or asks to author, configure, run, or diagnose `.dcua` tests with the Droid CUA CLI or desktop app. Do not use for general web browsing, website development, browser automation, generic mobile work, or non-Droid test tasks.
 ---
 
 # Droid CUA
@@ -13,13 +13,16 @@ Use the existing `droid-cua` CLI and repository files. Do not invent commands, f
 2. Determine the requested platform and target from the task or project. Ask the user only when the choice is unclear.
 3. Check authentication and environment readiness before the first run. Read [project-and-auth.md](references/project-and-auth.md) for project boundaries and token handling. Read [onboarding-and-troubleshooting.md](references/onboarding-and-troubleshooting.md) when setup, dependency, device, simulator, browser, or connection work is needed.
 4. Author or update the smallest useful test. Read [authoring-tests.md](references/authoring-tests.md) before changing `.dcua`, context, test data, secrets, or config.
-5. Run the narrowest relevant test and monitor it to completion. Read [running-and-healing.md](references/running-and-healing.md) for commands, targets, artifacts, failure classification, and guarded healing.
-6. Report the outcome, evidence paths, changes made, and whether the failure belongs to the test, environment, or product.
+5. Before execution, present the proposed tests and target to the user, resolve any questions, and get their confirmation. Read [running-and-healing.md](references/running-and-healing.md) for the required pre-run check.
+6. Run the confirmed, narrowest relevant test and monitor it to completion. Keep the user oriented during the run by surfacing meaningful live screenshots, not just terminal output.
+7. Report the outcome, evidence paths, changes made, and whether the failure belongs to the test, environment, or product.
 
 ## Safety
 
 - Never print, open, or echo secret values. Check only whether required keys are configured.
 - Never search for credentials outside the active test project or source another project's `.env` file.
+- Use Loadmill for every run unless the user explicitly asks to use their own OpenAI API key. A discovered OpenAI key is never permission to switch providers.
+- Never use credentials from another project, a skill, an agent configuration, or the user's home directory. Configure credentials only in the active test project's `.env` (or an explicitly supplied CI secret).
 - Get approval before installing dependencies, running `sudo`, accepting licenses, or changing system configuration.
 - During setup, do not create or edit user tests. Rerun only the failed readiness check, then resume the original task.
 - During healing, edit only `.dcua`, `context.md`, `test-data.md`, or Droid CUA config. Never modify application code under the guise of healing a test.

package/skill/droid-cua/agents/openai.yaml

@@ -1,6 +1,6 @@
 interface:
   display_name: "Droid Cua"
-  short_description: "Write, run, monitor, and troubleshoot Droid CUA tests"
+  short_description: "Operate Droid CUA .dcua test projects"
   icon_small: "./assets/droid-cua-app-icon.png"
   icon_large: "./assets/droid-cua-app-icon.png"
   brand_color: "#1895F7"

package/skill/droid-cua/references/project-and-auth.md

@@ -15,24 +15,27 @@ Keep `.env` and `.secrets` out of version control. Use `git check-ignore -q .env
 
 ## Loadmill authentication
 
-Loadmill is the default LLM provider. Before a run, check only for presence:
+Loadmill is the required default LLM provider. Use it unless the user explicitly asks to use their own OpenAI API key. Never switch to OpenAI because a Loadmill token is missing or because an OpenAI key happens to be available.
+
+Before a run, check only for the presence of a Loadmill token in the active test project's `.env` or explicitly supplied CI secret:
 
 ```sh
-test -n "${LOADMILL_API_TOKEN:-}" || grep -Eq '^LOADMILL_API_TOKEN=.+$' .env 2>/dev/null
+grep -Eq '^LOADMILL_API_TOKEN=.+$' .env 2>/dev/null
 ```
 
-Run this check from the active test project's working directory. Do not print the variable, run `env`, open `.env`, or ask the user to paste the token. Do not search parent directories, sibling repositories, the user's home directory, or the Droid CUA source repository for credentials.
+Run this check from the active test project's working directory. Do not print the variable, run `env`, open `.env`, or inspect inherited environment variables. Do not search parent directories, sibling repositories, the user's home directory, agent skills/configuration, or the Droid CUA source repository for credentials.
+
+If it is missing, explain that Droid CUA uses a Loadmill token by default and guide the user to:
+
+1. Sign in, then open [Loadmill Settings → Security](https://app.loadmill.com/app/user/settings/security).
+2. Create an API token with a description such as `Droid CUA CLI`.
+3. Copy it when shown; Loadmill shows it once.
 
-If it is missing, instruct the user to:
+Offer exactly these two choices, then wait for the user's selection:
 
-1. Sign in at <https://app.loadmill.com>.
-2. Open **Settings → Security**.
-3. Create an API token with a description such as `Droid CUA CLI`.
-4. Copy it when shown; Loadmill shows it once.
-5. Ensure `.env` is ignored by version control before storing a project token; add `.env` to `.gitignore` if needed.
-6. Save it as `LOADMILL_API_TOKEN=...` in the active test project's `.env`, a shell environment variable, or a CI secret.
-7. Confirm when configuration is complete, without sharing the value.
+1. **Use it once:** The user provides the token through the agent's secret-input mechanism and asks to use it for this run only. Pass it only to that command through a secret-safe mechanism; do not write it to `.env`, a shell profile, or any other file.
+2. **Save it for this project:** The user provides the token through the agent's secret-input mechanism and approves saving it. Create the active project's `.env` if needed or add/update only its `LOADMILL_API_TOKEN` entry. First ensure `.env` is ignored by version control; add `.env` to the active project's `.gitignore` when needed.
 
-Do not create or populate the token value for the user. Do not copy a token from another file or source another project's `.env`. Sourcing an unrelated file can import development endpoints, proxy settings, and unrelated secrets. When the active project's `.env` contains the key, run `droid-cua` normally from that project; the CLI loads that file automatically.
+Use a secret-safe mechanism that does not print or expose the value in commands, logs, or responses. Verify only that the key is present. Never copy a token from another file, a shell profile, another project, an agent skill/configuration, or any other location. Do not inspect inherited environment variables to discover a token. When the active project's `.env` contains the key, run `droid-cua` normally from that project; the CLI loads that file automatically.
 
-If the user explicitly selects direct OpenAI access, use `--llm-provider openai` with one of the supported OpenAI CUA models: `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Check only for the presence of `OPENAI_API_KEY`. If it is missing, instruct the user to create an API key in their OpenAI account and save it in the active project's gitignored `.env`, their shell environment, or a CI secret. Apply the same secret-handling rules: never request, print, inspect, copy, or source the key. Do not silently switch providers to bypass missing Loadmill authentication.
+Only when the user explicitly selects direct OpenAI access, use `--llm-provider openai` with one of the supported OpenAI CUA models: `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Check only the active project's `.env` for `OPENAI_API_KEY`; do not use one found elsewhere. If it is missing, guide the user to create and configure their own key in the same project using the same secret-safe process. Do not silently switch providers to bypass missing Loadmill authentication.

package/skill/droid-cua/references/running-and-healing.md

@@ -14,7 +14,7 @@ Prefer a single test before a directory or scenario run. Preserve an existing pr
 
 Both Loadmill models require `LOADMILL_API_TOKEN`. Do not switch between Smart and Beacon silently; preserve the project's model or explain the speed/cost tradeoff and follow the user's choice. If the user chooses direct OpenAI access instead, require `OPENAI_API_KEY` and use `--llm-provider openai` with one of `gpt-5.6-terra`, `gpt-5.6-luna`, or `gpt-5.4`. Do not mix a provider with another provider's model, silently switch providers, or override an existing project choice without explaining it.
 
-Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` when diagnosis may be needed.
+Include `--report reports/<test-name>.html` for normal test runs unless the user opts out. Prefer an explicit filename over a directory so the result is easy to locate. Add `--debug` for every agent-observed run: it writes the screenshots needed for live progress updates as well as diagnosis.
 
 ```sh
 # Attached Android device or running emulator
@@ -41,9 +41,49 @@ droid-cua --llm-provider openai --cua-model gpt-5.4 --avd adb:<serial> --instruc
 
 Use `.ipa` and `--platform ios` for supported cloud iOS runs. Preserve existing config flags rather than replacing them with defaults.
 
+## Confirm the run with the user
+
+Before starting any test command, send the user a short, plain-language run plan and wait for explicit confirmation. Treat this as a product conversation, not a command preview. Do not infer that a request to run a test also confirms an automatically selected device, browser, or batch of tests.
+
+The run plan should briefly state:
+
+- What behavior or flow will be checked, and whether one test or several will run. Mention the test filename only when it helps identify the test.
+- Where it will run: the chosen device, simulator, or browser. If more than one viable target is available, present the human-readable choices and ask the user to choose.
+- Any user-visible consequence, such as launching an emulator, resetting a fresh browser session, or starting a metered cloud-device session.
+- That the agent will share progress screenshots and provide a report afterward.
+
+Do not include CLI flags, commands, provider or model names, credential/key names, config/context filenames, or internal log/report paths unless the user asks or one changes a decision they need to make. Do not paste the test source into the confirmation.
+
+Ask the user about every unresolved or ambiguous choice before running. If the plan is complete, ask for a clear go-ahead (for example, `Ready to run this?`). Start only after they confirm. If they change the tests, target, or other material run detail, show the revised plan and confirm it again.
+
+For example: `I’m ready to run the Chrome smoke test in a fresh local Chrome window. It will check that Chrome opens and then closes. I’ll share screenshots as it progresses and provide a report at the end. Ready to run this?`
+
 ## Monitor and collect evidence
 
-Remain attached until the command exits and cloud cleanup finishes. Read terminal progress as it appears; do not treat a long model step as success or failure before completion.
+Remain attached until the command exits and cloud cleanup finishes. Run the command through an interactive/pollable terminal mechanism so progress and new artifacts can be observed before it exits. Read terminal progress as it appears; do not treat a long model step as success or failure before completion.
+
+## Show live visual progress
+
+Use the screenshots already captured by `--debug` to make the run understandable while it is happening. This is a required user-facing behavior, not merely an artifact to inspect after the run. After the CLI prints `Debug logging enabled: <logs/execution-...>.jsonl`, new PNGs appear in that log file's sibling directory. Keep the CLI running in a pollable/background session and poll that directory while the test is running.
+
+For agents whose chat UI renders local Markdown images (including Cursor and Codex), send a separate progress message containing an absolute-path image, for example:
+
+```md
+Step 2/4: add the item to the cart
+
+![Live test progress](/absolute/path/to/logs/execution-.../0004_....png)
+```
+
+Do not put this Markdown only in a terminal command, tool argument, or hidden reasoning: it must be an actual user-visible chat update. Do not replace it by opening the final report.
+
+- After every completed executable instruction, send exactly one short progress update with that instruction and its screenshot (for example, `Step 2/4 complete: add the item to the cart`). An image path alone is not a visual update.
+- Screenshot filenames include `instruction-001`, `instruction-002`, and so on. Select the latest safe PNG for the completed instruction; prefer an end-of-turn or assertion screenshot over an input or click-preview frame. This connects each user-visible update to the matching script step even when the model took several actions to finish it.
+- Also show a frame immediately when the run is retrying, reconnecting, or fails. Do not stream every model action or repeat visually identical frames.
+- Do not show `preview_click` / aim-assist annotations unless they help explain an issue.
+- If the latest frame visibly exposes a secret or other sensitive user data, do not render it; provide a text-only progress update and wait for a safe frame.
+- Do not delay the test to collect screenshots. If no new frame exists during a long model call, report the current stage in text and keep polling.
+
+When the command completes, show the final safe screenshot in addition to the result summary, then provide a clickable link to the HTML report (or its absolute path when local links are unavailable). The report is the final artifact; the interim frames are progress updates, not a substitute for it.
 
 - Exit `0`: all requested tests passed.
 - Exit `1`: configuration, connection, execution, or assertion failure.