@loadmill/droid-cua 2.23.0 → 2.25.0

package/build/src/device/android/actions.js

@@ -6,9 +6,25 @@ import { buildMobileKeypressPayload, getAndroidAdbTransport, normalizeMobileKeyp
 import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
 import { execAdb } from "./tools.js";
+import { closeAdbKeyboard, resolveAdbKeyboardVisibility } from "../keyboard.js";
 function adbShell(deviceId, command) {
     return execAdb(["-s", deviceId, "shell", command]);
 }
+async function getKeyboardVisibility(deviceId) {
+    const { stdout: inputMethodDump } = await adbShell(deviceId, "dumpsys input_method");
+    const { stdout: showImeWithHardwareKeyboard } = await adbShell(deviceId, "settings get secure show_ime_with_hard_keyboard");
+    return resolveAdbKeyboardVisibility(inputMethodDump, showImeWithHardwareKeyboard);
+}
+export async function closeKeyboard(deviceId) {
+    // KEYCODE_BACK dismisses the Android IME, but it would navigate if the app
+    // already hid the keyboard. getKeyboardVisibility also filters the hardware
+    // keyboard configuration where mInputShown is true without a software input
+    // view. closeAdbKeyboard invokes Back only for an affirmative result.
+    return closeAdbKeyboard({
+        getVisibility: () => getKeyboardVisibility(deviceId),
+        pressBack: () => adbShell(deviceId, "input keyevent KEYCODE_BACK"),
+    });
+}
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const meta = (payload = {}) => buildActionEventMeta({

package/build/src/device/cloud/actions.js

@@ -6,6 +6,33 @@ import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSe
 import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
+import { closeAppiumKeyboard, closeAppiumKeyboardWithTap } from "../keyboard.js";
+export async function closeKeyboard(_deviceId, { dismissalPoint = null, scale = 1 } = {}) {
+    const session = getActiveSession();
+    if (!session?.client) {
+        throw new Error("No active cloud session");
+    }
+    if (session.platform === "ios") {
+        // XCUITest hideKeyboard was unreliable on the tested AWS real devices, and
+        // Return/newline can submit forms. Trust the model to identify a dismissal
+        // control or inert background, but tap only after confirming the keyboard
+        // is open. With no trusted point, fail safely without touching the screen.
+        const dpr = getDevicePixelRatio();
+        const tap = dismissalPoint
+            ? () => session.client.tap(session.sessionId, Math.round((dismissalPoint.x / scale) / dpr), Math.round((dismissalPoint.y / scale) / dpr))
+            : null;
+        return closeAppiumKeyboardWithTap({
+            isKeyboardShown: () => session.client.isKeyboardShown(session.sessionId),
+            tap,
+        });
+    }
+    // UiAutomator2's native query/hide commands are reliable on cloud Android and
+    // avoid the navigation risk of treating Android Back as a generic fallback.
+    return closeAppiumKeyboard({
+        isKeyboardShown: () => session.client.isKeyboardShown(session.sessionId),
+        hideKeyboard: () => session.client.hideKeyboard(session.sessionId),
+    });
+}
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const session = getActiveSession();

package/build/src/device/cloud/cloud-appium-client.js

@@ -99,6 +99,21 @@ export class CloudAppiumClient {
             ]
         });
     }
+    async isKeyboardShown(sessionId) {
+        const response = await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: isKeyboardShown",
+            args: [{}]
+        });
+        return response.value === true;
+    }
+    async hideKeyboard(sessionId) {
+        // Currently called by cloud Android only. Cloud iOS deliberately routes to
+        // its guarded model-selected tap in cloud/actions.js.
+        await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: hideKeyboard",
+            args: [{}]
+        });
+    }
     async scroll(sessionId, startX, startY, endX, endY, duration = 500) {
         await this.request("POST", `/session/${sessionId}/actions`, {
             actions: [

package/build/src/device/connection.js

@@ -4,7 +4,7 @@
  * Thin wrapper that delegates to the appropriate platform backend.
  * Maintains backwards compatibility with existing code.
  */
-import { getDeviceBackend, detectPlatform, setCurrentPlatform, setCurrentSource, getCurrentPlatform } from "./factory.js";
+import { getDeviceBackend, detectPlatform, setCurrentPlatform, setCurrentSource, getCurrentPlatform, getCurrentSource } from "./factory.js";
 let currentBackend = null;
 /**
  * Connect to a device (Android target ID/serial or iOS simulator name)
@@ -40,7 +40,10 @@ export async function getDeviceInfo(deviceId, options = {}) {
     if (!currentBackend) {
         throw new Error("Not connected to a device. Call connectToDevice first.");
     }
-    return currentBackend.getDeviceInfo(deviceId, options);
+    return {
+        ...await currentBackend.getDeviceInfo(deviceId, options),
+        source: getCurrentSource(),
+    };
 }
 /**
  * Get screenshot as base64 string

package/build/src/device/factory.js

@@ -47,6 +47,7 @@ export function getDeviceBackend(platform) {
             getDeviceInfo: cloudConnection.getDeviceInfo,
             getScreenshotAsBase64: cloudConnection.getScreenshotAsBase64,
             handleModelAction: cloudActions.handleModelAction,
+            closeKeyboard: cloudActions.closeKeyboard,
             disconnect: cloudConnection.disconnect,
         };
     }
@@ -56,6 +57,7 @@ export function getDeviceBackend(platform) {
             getDeviceInfo: iosConnection.getDeviceInfo,
             getScreenshotAsBase64: iosConnection.getScreenshotAsBase64,
             handleModelAction: iosActions.handleModelAction,
+            closeKeyboard: iosActions.closeKeyboard,
             disconnect: iosConnection.disconnect,
         };
     }
@@ -65,6 +67,7 @@ export function getDeviceBackend(platform) {
         getDeviceInfo: androidConnection.getDeviceInfo,
         getScreenshotAsBase64: androidConnection.getScreenshotAsBase64,
         handleModelAction: androidActions.handleModelAction,
+        closeKeyboard: androidActions.closeKeyboard,
         disconnect: async () => { }, // Android doesn't need explicit disconnect
     };
 }

package/build/src/device/interface.js

@@ -14,6 +14,7 @@
  * @property {string} [platform] - Device platform ('android' or 'ios')
  * @property {string} [device_name] - Human-readable device or simulator name
  * @property {string} [model] - Device model identifier/name
+ * @property {string} [source] - Device source such as 'local' or a cloud provider ID
  */
 /**
  * @typedef {Object} ActionContext
@@ -41,6 +42,9 @@
  *   - Executes an action from the CUA model
  *   - Supported action types: click, type, scroll, drag, keypress, wait
  *   - keypress is limited to single mobile-safe keys only; desktop shortcut chords are unsupported
+ *
+ * closeKeyboard(deviceId: string, options?: {dismissalPoint?: {x: number, y: number}, scale?: number}): Promise<{status: string, message: string, error?: string}>
+ *   - Safely attempts to close the software keyboard without changing app navigation
  */
 export const SUPPORTED_ACTIONS = [
     'click', // Tap at (x, y) coordinates

package/build/src/device/ios/actions.js

@@ -12,6 +12,19 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
+import { closeAppiumKeyboard } from "../keyboard.js";
+export async function closeKeyboard() {
+    const session = getActiveSession();
+    if (!session) {
+        throw new Error("No active iOS session");
+    }
+    // The local simulator supports Appium's keyboard command reliably, so it
+    // needs neither a model-selected tap nor a direct Return keypress.
+    return closeAppiumKeyboard({
+        isKeyboardShown: () => appium.isKeyboardShown(session.sessionId),
+        hideKeyboard: () => appium.hideKeyboard(session.sessionId),
+    });
+}
 /**
  * Handle an action from the CUA model
  * @param {string} simulatorId - The simulator UDID

package/build/src/device/ios/appium-client.js

@@ -123,6 +123,22 @@ export async function type(sessionId, text) {
         ],
     });
 }
+export async function isKeyboardShown(sessionId) {
+    const response = await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: isKeyboardShown",
+        args: [{}],
+    });
+    return response.value === true;
+}
+export async function hideKeyboard(sessionId) {
+    // This XCUITest form is used only by the locally validated simulator path.
+    // Real cloud iOS devices use a guarded tap instead because this command was
+    // unreliable there and sending Return directly can submit the current form.
+    await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: hideKeyboard",
+        args: [{ keys: ["return"] }],
+    });
+}
 /**
  * Perform a scroll action using W3C Actions
  * @param {string} sessionId

package/build/src/device/keyboard.js

@@ -0,0 +1,119 @@
+import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+/**
+ * Keyboard dismissal is intentionally platform-specific:
+ *
+ * - Local Android uses ADB. Back is the reliable dismissal key, but it can also
+ *   navigate away. Android's mInputShown flag stays true when a hardware
+ *   keyboard suppresses the software keyboard, so that configuration is
+ *   treated as already closed before Back is considered.
+ * - Local iOS and cloud Android use Appium's native keyboard commands, which
+ *   worked reliably in device testing and do not require choosing a tap target.
+ * - Cloud iOS uses a model-selected tap. XCUITest's hideKeyboard command was
+ *   unreliable on the tested real devices, while Return/newline could submit a
+ *   form. The caller therefore supplies a trusted dismissal point instead.
+ *
+ * Every strategy follows the same query -> dismiss -> wait -> verify shape and
+ * returns the normalized result below.
+ */
+export const KEYBOARD_CLOSE_RESULTS = {
+    CLOSED: "closed",
+    ALREADY_CLOSED: "already_closed",
+    FAILED: "failed",
+};
+const KEYBOARD_CLOSE_MESSAGES = {
+    [KEYBOARD_CLOSE_RESULTS.CLOSED]: "Keyboard closed",
+    [KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED]: "Keyboard closed",
+    [KEYBOARD_CLOSE_RESULTS.FAILED]: "Keyboard could not be closed and may still be open",
+};
+export function buildKeyboardCloseResult(status, error = null) {
+    return {
+        status,
+        message: KEYBOARD_CLOSE_MESSAGES[status],
+        ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}),
+    };
+}
+export function parseAdbKeyboardVisibility(output) {
+    const matches = [...String(output ?? "").matchAll(/\bmInputShown=(true|false)\b/g)]
+        .map((match) => match[1] === "true");
+    const states = new Set(matches);
+    // Multiple contradictory sections can appear while the IME is transitioning.
+    // Treat missing or ambiguous state as unknown so callers never risk Back.
+    return states.size === 1 ? matches[0] : null;
+}
+export function parseAdbHardwareKeyboardVisibility(output) {
+    const matches = [...String(output ?? "").matchAll(/\b(?:qwerty|12key)\/[vh]\/(v|h)\b/g)]
+        .map((match) => match[1] === "v");
+    const states = new Set(matches);
+    return states.size === 1 ? matches[0] : null;
+}
+export function resolveAdbKeyboardVisibility(inputMethodDump, showImeWithHardwareKeyboard) {
+    const softwareKeyboardRequested = parseAdbKeyboardVisibility(inputMethodDump);
+    const hardwareKeyboardVisible = parseAdbHardwareKeyboardVisibility(inputMethodDump);
+    const showSoftwareKeyboard = String(showImeWithHardwareKeyboard ?? "").trim();
+    // mInputShown means Android asked the IME to show; it does not guarantee that
+    // the input view appeared. With an exposed hardware keyboard and this setting
+    // disabled, only an IME accessory toolbar may be visible, and Back would be
+    // delivered to the app instead of dismissing a software keyboard.
+    if (showSoftwareKeyboard === "0" && hardwareKeyboardVisible === true) {
+        return false;
+    }
+    return softwareKeyboardRequested;
+}
+async function waitForKeyboardUpdate() {
+    const delayMs = getConfiguredStepDelayMs();
+    if (delayMs > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+}
+export async function closeAppiumKeyboard({ isKeyboardShown, hideKeyboard }) {
+    try {
+        if (!await isKeyboardShown()) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED);
+        }
+        await hideKeyboard();
+        await waitForKeyboardUpdate();
+        return await isKeyboardShown()
+            ? buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED)
+            : buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.CLOSED);
+    }
+    catch (error) {
+        return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+}
+export async function closeAppiumKeyboardWithTap({ isKeyboardShown, tap }) {
+    try {
+        if (!await isKeyboardShown()) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED);
+        }
+        if (typeof tap !== "function") {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED);
+        }
+        await tap();
+        await waitForKeyboardUpdate();
+        return await isKeyboardShown()
+            ? buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED)
+            : buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.CLOSED);
+    }
+    catch (error) {
+        return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+}
+export async function closeAdbKeyboard({ getVisibility, pressBack }) {
+    try {
+        const visibility = await getVisibility();
+        if (visibility === false) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.ALREADY_CLOSED);
+        }
+        if (visibility !== true) {
+            return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED);
+        }
+        await pressBack();
+        await waitForKeyboardUpdate();
+        return await getVisibility() === false
+            ? buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.CLOSED)
+            : buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED);
+    }
+    catch (error) {
+        return buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+}

package/build/src/device/openai.js

@@ -99,6 +99,21 @@ const closeCurrentTabTool = {
         additionalProperties: false,
     },
 };
+const closeKeyboardTool = {
+    type: "function",
+    name: "close_keyboard",
+    description: "Close the mobile software keyboard after text entry. Call this when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden. Do not call it for a slim hardware-keyboard toolbar or suggestions alone. On cloud iOS, optionally provide x and y for a dismissal control or clearly noninteractive background that you trust; never target Return, Go, Search, Next, Continue, Send, or an app action.",
+    strict: false,
+    parameters: {
+        type: "object",
+        properties: {
+            x: { type: "number", description: "Optional horizontal coordinate in the current screenshot." },
+            y: { type: "number", description: "Optional vertical coordinate in the current screenshot." },
+        },
+        required: [],
+        additionalProperties: false,
+    },
+};
 const loadmillTool = {
     type: "function",
     name: "loadmill",
@@ -116,7 +131,7 @@ const loadmillTool = {
 const typeSensitiveTextTool = {
     type: "function",
     name: "type_sensitive_text",
-    description: "Type a secret value by key without exposing the value to the model. Use this only for secret keys listed in the prompt.",
+    description: "Type a secret value by key without exposing the value to the model. Use this only for secret keys listed in the prompt. On mobile, apply the prompt's keyboard-dismissal rules afterward.",
     strict: true,
     parameters: {
         type: "object",
@@ -175,6 +190,9 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
     if (toolCapabilities.closeCurrentTab) {
         tools.push(closeCurrentTabTool);
     }
+    if (toolCapabilities.closeKeyboard) {
+        tools.push(closeKeyboardTool);
+    }
     if (toolCapabilities.loadmill) {
         tools.push(loadmillTool);
     }

package/build/src/modes/design-mode-ink.js

@@ -41,6 +41,7 @@ export class DesignModeInk {
             assert: true,
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
+            closeKeyboard: this.session?.deviceInfo?.platform !== "browser",
             loadmill: false,
             typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,

package/build/src/modes/design-mode.js

@@ -38,6 +38,7 @@ export class DesignMode {
             assert: true,
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
+            closeKeyboard: this.session?.deviceInfo?.platform !== "browser",
             loadmill: false,
             typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,

package/build/src/modes/execution-mode.js

@@ -1,4 +1,5 @@
 import { CAPTURE_SOURCES } from "../core/capture-sources.js";
+import { CUA_CONVERSATION_POLICIES, resolveConversationPolicy } from "../core/cua-models.js";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
 import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, } from "../browser/connection.js";
 import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
@@ -22,7 +23,8 @@ const LLM_USAGE_LIMIT_ERROR_CODES = new Set([
 ]);
 const LOADMILL_UPGRADE_URL = "https://www.loadmill.com/book-a-demo";
 const LLM_REQUEST_RATE_LIMIT_ERROR_CODE = "llm_rate_limit_exceeded";
-const DEFAULT_ASSERTION_CONTEXT = "No additional prior context is needed; validate directly from the current screen.";
+const OPENAI_INSUFFICIENT_QUOTA_ERROR_CODE = "insufficient_quota";
+const DEFAULT_ASSERTION_CONTEXT = "Use the session transcript to resolve references to prior scenario steps.";
 function isLlmUsageLimitError(err) {
     if (!err || typeof err !== "object") {
         return false;
@@ -35,15 +37,31 @@ function isLlmUsageLimitError(err) {
 function isLlmRequestRateLimitError(err) {
     return err && typeof err === "object" && err.code === LLM_REQUEST_RATE_LIMIT_ERROR_CODE;
 }
+function isOpenAiInsufficientQuotaError(err) {
+    return err && typeof err === "object" && err.code === OPENAI_INSUFFICIENT_QUOTA_ERROR_CODE;
+}
+function isRejectedContinuationError(err) {
+    if (!err || typeof err !== "object") {
+        return false;
+    }
+    return err.param === "previous_response_id" || [
+        "provider_continuation_mismatch",
+        "unknown_previous_response_id",
+        "invalid_previous_response_id",
+    ].includes(err.code);
+}
 function getLlmUsageLimitMessage() {
     return `Execution stopped: you've reached the AI usage limit for this account. Contact the Loadmill team to upgrade: ${LOADMILL_UPGRADE_URL}`;
 }
 function getLlmRequestRateLimitMessage() {
     return "Execution stopped: too many AI requests are running for this account right now. Please retry shortly or reduce parallel test runs.";
 }
+function getOpenAiInsufficientQuotaMessage() {
+    return "Execution stopped: your OpenAI account has insufficient quota. Check your OpenAI plan and billing details.";
+}
 /**
  * Execution Mode - Run test scripts line-by-line
- * Each instruction is executed in isolation (messages cleared after each turn)
+ * Instructions share scenario context unless the selected model requires isolation.
  */
 export class ExecutionMode {
     constructor(session, executionEngine, instructions, isHeadlessMode = false) {
@@ -77,6 +95,7 @@ export class ExecutionMode {
             assert: true,
             navigate: this.isBrowserSession,
             closeCurrentTab: this.isBrowserSession,
+            closeKeyboard: !this.isBrowserSession,
             loadmill: true,
             typeSensitiveText: this.engine.hasSecrets(),
         };
@@ -132,8 +151,9 @@ export class ExecutionMode {
             payload: extra.payload
         });
     }
-    resetInstructionConversationState() {
+    resetScenarioConversationState() {
         this.session.updateResponseId(null);
+        this.session.clearMessages();
         if (typeof this.session.clearTranscript === "function") {
             this.session.clearTranscript();
             return;
@@ -142,17 +162,53 @@ export class ExecutionMode {
             this.session.transcript = [];
         }
     }
-    async executeVerificationInstruction(instruction, context, stepContext = null) {
-        const toolResult = await handleAssertionToolCall({
-            engine: this.engine,
-            callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
-            argumentsText: JSON.stringify({
-                condition: extractAssertionPrompt(instruction),
-                context: DEFAULT_ASSERTION_CONTEXT,
-            }),
-            context,
-            stepContext,
+    startInstructionConversation(context) {
+        const conversationPolicy = resolveConversationPolicy(context?.runtimeConfig?.cuaModel);
+        if (conversationPolicy === CUA_CONVERSATION_POLICIES.ISOLATED) {
+            this.session.updateResponseId(null);
+        }
+    }
+    stopForOpenAiInsufficientQuota(err, context, stepContext) {
+        const addOutput = context.addOutput || printCliOutput;
+        const stopMessage = getOpenAiInsufficientQuotaMessage();
+        logger.debug('Stopping execution because the OpenAI account has insufficient quota', {
+            instruction: context?.currentInstruction,
+            message: err.message,
+            status: err.status,
+            code: err.code,
+            type: err.type,
+        });
+        this.emit(addOutput, 'error', stopMessage, context, stepContext, {
+            eventType: 'error',
+            payload: {
+                message: stopMessage,
+                status: err.status,
+                code: err.code,
+                type: err.type,
+            }
         });
+        return { success: false, error: stopMessage, usageLimitReached: true };
+    }
+    async executeVerificationInstruction(instruction, context, stepContext = null) {
+        let toolResult;
+        try {
+            toolResult = await handleAssertionToolCall({
+                engine: this.engine,
+                callId: stepContext?.stepId ? `${stepContext.stepId}-assert` : "instruction-assert",
+                argumentsText: JSON.stringify({
+                    condition: extractAssertionPrompt(instruction),
+                    context: DEFAULT_ASSERTION_CONTEXT,
+                }),
+                context,
+                stepContext,
+            });
+        }
+        catch (err) {
+            if (isOpenAiInsufficientQuotaError(err)) {
+                return this.stopForOpenAiInsufficientQuota(err, context, stepContext);
+            }
+            throw err;
+        }
         const assertionStatus = toolResult.summaryUpdates?.assertionStatus;
         if (assertionStatus === "passed") {
             this.stats.assertionsPassed++;
@@ -184,6 +240,7 @@ export class ExecutionMode {
         const addOutput = runContext.addOutput || printCliOutput;
         // Start timing
         this.stats.startTime = Date.now();
+        this.resetScenarioConversationState();
         for (let i = 0; i < this.instructions.length; i++) {
             const instruction = this.instructions[i];
             const stepContext = this.buildStepContext(i, instruction);
@@ -288,7 +345,7 @@ export class ExecutionMode {
             futureInstructions: this.instructions.slice((stepContext?.instructionIndex ?? -1) + 1),
         };
         if (retryCount === 0) {
-            this.resetInstructionConversationState();
+            this.startInstructionConversation(context);
         }
         this.session.addToTranscript(`[User] ${instruction}`);
         this.session.addMessage("user", instruction);
@@ -364,6 +421,18 @@ export class ExecutionMode {
                 stack: err.stack
             });
             const addOutput = context.addOutput || printCliOutput;
+            if (this.session.previousResponseId && isRejectedContinuationError(err)) {
+                this.emit(addOutput, 'error', err.message, context, stepContext, {
+                    eventType: 'error',
+                    payload: {
+                        message: err.message,
+                        status: err.status,
+                        code: err.code,
+                        param: err.param,
+                    }
+                });
+                return { success: false, error: err.message };
+            }
             if (isLlmRequestRateLimitError(err)) {
                 const stopMessage = getLlmRequestRateLimitMessage();
                 logger.debug('Stopping execution because an LLM request rate limit was reached', {
@@ -384,6 +453,9 @@ export class ExecutionMode {
                 });
                 return { success: false, error: stopMessage, usageLimitReached: true };
             }
+            if (isOpenAiInsufficientQuotaError(err)) {
+                return this.stopForOpenAiInsufficientQuota(err, context, stepContext);
+            }
             if (isLlmUsageLimitError(err)) {
                 const stopMessage = getLlmUsageLimitMessage();
                 logger.debug('Stopping execution because an LLM usage limit was reached', {
@@ -448,8 +520,8 @@ export class ExecutionMode {
                     }
                     else {
                         const platform = this.getCurrentPlatform();
-                        const deviceName = this.session.deviceName || undefined;
-                        const deviceId = await this.connectToDevice(deviceName, platform);
+                        const deviceTarget = this.session.deviceId || this.session.deviceName || undefined;
+                        const deviceId = await this.connectToDevice(deviceTarget, platform);
                         const deviceInfo = await this.getDeviceInfo(deviceId);
                         this.session.deviceId = deviceId;
                         this.session.deviceInfo = deviceInfo;

package/build/src/prompts/base.js

@@ -30,6 +30,11 @@ ${customSection}
 function isBrowserTarget(deviceInfo = {}) {
     return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
 }
+function isCloudIosTarget(deviceInfo = {}) {
+    const platform = deviceInfo.platform?.trim().toLowerCase();
+    const source = deviceInfo.source?.trim().toLowerCase() || "local";
+    return platform === "ios" && source !== "local";
+}
 function describeControlledTarget(deviceInfo = {}) {
     const platform = typeof deviceInfo.platform === "string" ? deviceInfo.platform.trim().toLowerCase() : "";
     const deviceName = typeof deviceInfo.device_name === "string" ? deviceInfo.device_name.trim() : "";
@@ -38,7 +43,8 @@ function describeControlledTarget(deviceInfo = {}) {
         return browserName ? `a browser session (${browserName})` : "a browser session";
     }
     if (platform === "ios") {
-        return deviceName ? `an iOS simulator (${deviceName})` : "an iOS device";
+        const target = isCloudIosTarget(deviceInfo) ? "an iOS device" : "an iOS simulator";
+        return deviceName ? `${target} (${deviceName})` : target;
     }
     if (platform === "android") {
         return deviceName ? `an Android device (${deviceName})` : "an Android device";
@@ -150,6 +156,12 @@ function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
 }
 function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
+    const cloudIosKeyboardGuidance = isCloudIosTarget(deviceInfo) ? `
+  - On this cloud iOS device, \`close_keyboard\` can optionally tap a point you choose from the current screenshot.
+  - If you can identify a dedicated keyboard dismissal control or clearly noninteractive background that you trust, pass its \`x\` and \`y\` coordinates to \`close_keyboard\`.
+  - Never use Return, Go, Search, Next, Continue, Send, or an app action as the dismissal point because it may submit or mutate the app.
+  - If dismissal is required but no safe point is identifiable, call \`close_keyboard\` without coordinates. Do not guess.
+  - Choose coordinates directly from the post-type screenshot; do not call \`preview_click\` between \`type\` and \`close_keyboard\`.` : "";
     return `
   You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
   Follow the user's instructions to interact with the device.
@@ -164,18 +176,31 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
   - Reason carefully about the approximate pixel position.
   ${buildAimAssistGuidance("mobile", aimAssistMode)}
 
-  Available actions: click, scroll, type, keypress, wait, screenshot.
+  Available actions: click, scroll, type, keypress, wait, screenshot. You also have a \`close_keyboard\` tool.
 
   CRITICAL - Mobile Input Constraints:
   - This is a mobile device, not a desktop. Do NOT use desktop keyboard shortcuts or modifier chords.
   - NEVER emit key combinations such as CTRL+A, CMD+A, CTRL+C, CTRL+V, ALT+TAB, SHIFT+ENTER, or similar shortcuts.
   - Use 'keypress' only for a single mobile-safe key when absolutely necessary.
   - To replace text, tap into the field and type the desired value. If correction is needed, use mobile-safe deletion only.
+  - Every \`type\` action must be the final computer action in its batch.
+  - After \`type\`, inspect its post-action screenshot before deciding whether dismissal is needed.
+  - If a full software keyboard is visibly covering part of the screen, your next action must be \`close_keyboard\`.
+  - Do not finish the current instruction while a required \`close_keyboard\` action is still pending.
+  - A slim hardware-keyboard toolbar, autocomplete suggestions, or a focused field without a full software keyboard does not require \`close_keyboard\`.
+  - Treat \`type_sensitive_text\` as text entry too and apply the same dismissal decision after it returns.
+  - Whenever dismissal is required, do not click, scroll, inspect, submit, type again, issue a keypress, or call another tool before \`close_keyboard\`.
+  - Consecutive DELETE or BACKSPACE keypresses may be used for correction without closing the keyboard between them. After typing the replacement, call \`close_keyboard\`.
+  - Password and other secure fields may appear empty in screenshots even after text was entered successfully.
+  - On iOS, a secure field may also hide an open keyboard from Appium screenshots. After typing into an iOS secure field, call \`close_keyboard\` even when the keyboard is not visible.
+  - Do not type into a secure field again solely because its characters or bullets are not visible.
+  - Retry secure-field input only when there is separate evidence of failure, such as another field changing or the app rejecting the value.
+  ${cloudIosKeyboardGuidance}
   - Prefer tapping visible controls over hardware key events.
   - Prefer on-screen navigation controls such as menus, tabs, drawer items, back arrows, close buttons, and explicit logout buttons over keypress actions.
   - Do NOT use Back or ESC for normal app navigation when a reliable on-screen control is visible.
   - Avoid using Back or ESC from a main or root screen, because it may leave the app.
-  - Exception: if the software keyboard is open and blocking the next needed control, Back or ESC may be used to dismiss the keyboard before continuing.
+  - Never use Back or ESC to dismiss the software keyboard; use \`close_keyboard\`.
   - Treat keypress actions as a fallback for limited cases only, such as a clearly needed single mobile-safe key or dismissing transient UI when no better visible control exists.
 
   CRITICAL - Automatic Timing:
@@ -185,7 +210,7 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
 
   CRITICAL - Mutating Actions:
   - Mutating actions change app state. Examples: submit, create, save, confirm, approve, reject, login, logout, send, place order, initiate transfer
-  - Before tapping a mutating action button, dismiss the software keyboard first when it is open and not required for the tap
+  - Before tapping a mutating action button, call \`close_keyboard\` first when the dismissal rules above require it
   - After performing a mutating action once, do NOT repeat the same mutating action unless the UI clearly shows the first attempt failed or had no effect
   - Treat visible state change as success. Examples: form fields clear, submit button returns to normal, status changes, list refreshes, new row appears, success message appears, screen changes
   - For form submissions specifically, if the relevant fields clear and the action button returns to its normal idle state, treat that as success even if the new row or confirmation is not obvious yet