@loadmill/droid-cua 2.24.0 → 2.26.0

package/build/src/commands/help.js

@@ -26,6 +26,8 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --device-name <name>   LambdaTest real device name' });
     addOutput({ type: 'info', text: '  --os-version <version> LambdaTest OS version' });
     addOutput({ type: 'info', text: '  --app <path>           LambdaTest app package path (.apk or .ipa)' });
+    addOutput({ type: 'info', text: '  --artifacts <kinds>    Loadmill Cloud: download video,deviceLog,appiumLog (or all) after the run' });
+    addOutput({ type: 'info', text: '  --cloud-session-id <id> Loadmill Cloud: reconnect/retroactive-fetch by mtc_ id (implies --device-source loadmill-cloud)' });
     addOutput({ type: 'info', text: '  --instructions <file>  Run test file in headless mode' });
     addOutput({ type: 'info', text: '  --scenario <prompt>    Generate and run scenario cases for headless instructions' });
     addOutput({ type: 'info', text: '  --test-data <file>     Optional test-data.md file for --scenario' });

package/build/src/commands/run.js

@@ -67,8 +67,7 @@ export async function handleRun(args, session, context) {
     if (context.setAgentWorking) {
         context.setAgentWorking(true, 'Executing test...');
     }
-    // Reset session state to prevent context leak from design mode
-    // Each test instruction should execute in isolation
+    // Reset session state to prevent context leak from design mode.
     session.updateResponseId(undefined);
     session.clearMessages();
     const appContext = await resolveExecutionAppContext({ contextPath: context.appContextPath });
@@ -89,7 +88,10 @@ export async function handleRun(args, session, context) {
     }
     // Set execution mode system prompt (replaces any design mode prompt)
     const availableSecretKeys = context.engine.getAvailableSecretKeys();
-    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, { availableSecretKeys });
+    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, {
+        availableSecretKeys,
+        cuaModel: context.runtimeConfig?.cuaModel,
+    });
     session.setSystemPrompt(executionPrompt);
     // Create execution mode
     const executionMode = new ExecutionMode(session, context.engine, instructions);

package/build/src/core/cua-models.js

@@ -1,6 +1,10 @@
 export const DEFAULT_OPENAI_CUA_MODEL = "gpt-5.4";
 export const DEFAULT_LOADMILL_CUA_MODEL = "loadmill-smart";
 export const DEFAULT_CUA_MODEL = DEFAULT_LOADMILL_CUA_MODEL;
+export const CUA_CONVERSATION_POLICIES = Object.freeze({
+    CHAINED: "chained",
+    ISOLATED: "isolated",
+});
 export const CUA_MODEL_OPTIONS = Object.freeze([
     Object.freeze({
         id: "loadmill-smart",
@@ -9,6 +13,13 @@ export const CUA_MODEL_OPTIONS = Object.freeze([
         name: "Loadmill - Smart",
         label: "Loadmill - Smart (recommended)",
     }),
+    Object.freeze({
+        id: "loadmill-pulse",
+        providerMode: "loadmill",
+        providerLabel: "Loadmill",
+        name: "Loadmill - Pulse",
+        label: "Loadmill - Pulse",
+    }),
     Object.freeze({
         id: "loadmill-beacon",
         providerMode: "loadmill",
@@ -70,3 +81,8 @@ export function resolveDefaultCuaModelForProvider(providerMode) {
         ? DEFAULT_LOADMILL_CUA_MODEL
         : DEFAULT_OPENAI_CUA_MODEL;
 }
+export function resolveConversationPolicy(cuaModel) {
+    return cuaModel === "loadmill-beacon" || cuaModel === "beacon"
+        ? CUA_CONVERSATION_POLICIES.ISOLATED
+        : CUA_CONVERSATION_POLICIES.CHAINED;
+}

package/build/src/core/droid-cua-run-usage.js

@@ -18,10 +18,13 @@ const MODEL_PRICING = Object.freeze({
     "gpt-5.4": Object.freeze({
         ...OPENAI_GPT_5_4_PRICING,
     }),
-    // Loadmill Smart currently routes to GPT-5.4 behind the Loadmill LLM gateway.
+    // Loadmill Smart currently routes to GPT-5.6 Terra behind the Loadmill LLM gateway.
     // Keep this explicit so pricing can be updated here if the backing model changes.
     "loadmill-smart": Object.freeze({
-        ...OPENAI_GPT_5_4_PRICING,
+        ...OPENAI_GPT_5_6_TERRA_PRICING,
+    }),
+    "loadmill-pulse": Object.freeze({
+        ...OPENAI_GPT_5_6_LUNA_PRICING,
     }),
 });
 function finiteNumber(value) {
@@ -50,6 +53,9 @@ export function normalizeDroidCuaCostModel(model) {
     if (model === "loadmill-smart" || model === "smart") {
         return "loadmill-smart";
     }
+    if (model === "loadmill-pulse" || model === "pulse") {
+        return "loadmill-pulse";
+    }
     return null;
 }
 export function createDroidCuaRunUsageTotals({ model = null } = {}) {

package/build/src/core/execution-engine.js

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
 import { closeCurrentTab as closeCurrentBrowserTab, getBrowserInfo, navigateToUrl as navigateBrowserToUrl } from "../browser/connection.js";
-import { handleModelAction } from "../device/actions.js";
+import { closeKeyboard, handleModelAction } from "../device/actions.js";
 import { sendCUARequest } from "../device/openai.js";
 import { executeLoadmillInstruction } from "../device/loadmill.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
@@ -14,6 +14,7 @@ import { ScreenshotStore } from "./screenshot-store.js";
 import { handlePreviewClick } from "./tool-handlers/preview-click.js";
 import { handleNavigateToolCall } from "./tool-handlers/navigate.js";
 import { handleCloseCurrentTabToolCall } from "./tool-handlers/close-current-tab.js";
+import { handleCloseKeyboardToolCall } from "./tool-handlers/close-keyboard.js";
 import { handleAssertionToolCall } from "./tool-handlers/assert.js";
 import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
 import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
@@ -176,6 +177,8 @@ function formatActionForRecovery(action) {
         }
         case "scroll":
             return "Scrolled";
+        case "close_keyboard":
+            return "Closed the keyboard";
         default:
             return null;
     }
@@ -240,6 +243,7 @@ export class ExecutionEngine {
         this.stepDelayMs = options.stepDelayMs ?? getConfiguredStepDelayMs();
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
+        this.closeKeyboard = options.closeKeyboard || closeKeyboard;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
         this.reviseTestScript = options.reviseTestScript || null;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
@@ -247,6 +251,7 @@ export class ExecutionEngine {
         this.navigateToUrl = options.navigateToUrl || navigateBrowserToUrl;
         this.closeCurrentTab = options.closeCurrentTab || closeCurrentBrowserTab;
         this.executeLoadmillInstruction = options.executeLoadmillInstruction || executeLoadmillInstruction;
+        this.connectToDevice = options.connectToDevice || null;
         this.toolCapabilities = options.toolCapabilities || {};
         this.secretsByKey = options.secretsByKey && typeof options.secretsByKey === "object" ? options.secretsByKey : {};
         this.baseSystemPrompt = options.baseSystemPrompt || session?.systemPrompt || null;
@@ -323,6 +328,8 @@ export class ExecutionEngine {
                 return await handleNavigateToolCall(handlerArgs);
             case "close_current_tab":
                 return await handleCloseCurrentTabToolCall(handlerArgs);
+            case "close_keyboard":
+                return await handleCloseKeyboardToolCall(handlerArgs);
             case "loadmill":
                 return await handleLoadmillToolCall(handlerArgs);
             case "type_sensitive_text":

package/build/src/core/runtime-config.js

@@ -7,6 +7,7 @@ export const LOADMILL_MODEL_ALIASES = Object.freeze({
     DEFAULT_AGENT_MODEL: "default-agent-model",
     FAST: "fast",
     SMART: "smart",
+    PULSE: "pulse",
     BEACON: "beacon",
 });
 const LOADMILL_MODEL_ALIAS_SET = new Set(Object.values(LOADMILL_MODEL_ALIASES));
@@ -90,9 +91,13 @@ export function inferLlmProviderModeFromCredentials(credentials = {}) {
 }
 export function resolveGatewayCuaModelAlias({ cuaModel } = {}) {
     const normalizedCuaModel = resolveCuaModel(cuaModel);
-    return normalizedCuaModel === "loadmill-beacon"
-        ? LOADMILL_MODEL_ALIASES.BEACON
-        : LOADMILL_MODEL_ALIASES.SMART;
+    if (normalizedCuaModel === "loadmill-pulse") {
+        return LOADMILL_MODEL_ALIASES.PULSE;
+    }
+    if (normalizedCuaModel === "loadmill-beacon") {
+        return LOADMILL_MODEL_ALIASES.BEACON;
+    }
+    return LOADMILL_MODEL_ALIASES.SMART;
 }
 export function resolveEffectiveCuaModel({ llmProviderMode = null, cuaModel = null, } = {}) {
     const providerMode = resolveLlmProviderMode(llmProviderMode);

package/build/src/core/session.js

@@ -45,7 +45,7 @@ export class Session {
         this.messages = [{ role: "system", content: prompt }];
     }
     /**
-     * Add a line to the transcript (for error recovery and logging)
+     * Add a line to the scenario transcript for assertions, recovery, and logging.
      */
     addToTranscript(line) {
         this.transcript.push(line);
@@ -57,7 +57,7 @@ export class Session {
         return this.transcript.join("\n");
     }
     /**
-     * Clear the per-instruction transcript.
+     * Clear the scenario transcript at a run boundary.
      */
     clearTranscript() {
         this.transcript = [];

package/build/src/core/tool-handlers/close-keyboard.js

@@ -0,0 +1,103 @@
+import { CAPTURE_SOURCES } from "../capture-sources.js";
+import { KEYBOARD_CLOSE_RESULTS, buildKeyboardCloseResult } from "../../device/keyboard.js";
+import { printCliOutput } from "../../utils/console-output.js";
+import { emitDesktopDebug } from "../../utils/desktop-debug.js";
+import { buildFunctionCallOutputItem, buildToolOutputContent, buildTextToolOutputItem, isDesignContext, parseJsonArguments, } from "./utils.js";
+function parseDismissalPoint(argumentsText, deviceInfo = {}) {
+    const parsed = parseJsonArguments(argumentsText || "{}");
+    if (!parsed.ok || !parsed.value || typeof parsed.value !== "object" || Array.isArray(parsed.value)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: expected a JSON object." };
+    }
+    const hasX = Object.prototype.hasOwnProperty.call(parsed.value, "x");
+    const hasY = Object.prototype.hasOwnProperty.call(parsed.value, "y");
+    if (!hasX && !hasY) {
+        return { ok: true, value: null };
+    }
+    if (!hasX || !hasY || !Number.isFinite(parsed.value.x) || !Number.isFinite(parsed.value.y)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: x and y must both be finite numbers." };
+    }
+    const point = { x: parsed.value.x, y: parsed.value.y };
+    const width = Number(deviceInfo.scaled_width);
+    const height = Number(deviceInfo.scaled_height);
+    // We can validate coordinate shape and bounds, but only the model can judge
+    // whether a cloud-iOS point is a safe dismissal target in the current UI.
+    if (point.x < 0 || point.y < 0 ||
+        (Number.isFinite(width) && point.x >= width) ||
+        (Number.isFinite(height) && point.y >= height)) {
+        return { ok: false, message: "Invalid close_keyboard arguments: the dismissal point is outside the current screenshot." };
+    }
+    return { ok: true, value: point };
+}
+export async function handleCloseKeyboardToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    if (session?.targetKind === "browser" || session?.deviceInfo?.platform === "browser") {
+        return {
+            kind: "continue",
+            outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: "close_keyboard is only supported for mobile runs." })),
+        };
+    }
+    const parsedPoint = parseDismissalPoint(argumentsText, session.deviceInfo);
+    if (!parsedPoint.ok) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, parsedPoint.message),
+        };
+    }
+    const dismissalPoint = parsedPoint.value;
+    let result;
+    try {
+        result = await engine.closeKeyboard(session.deviceId, {
+            dismissalPoint,
+            scale: session.deviceInfo?.scale || 1,
+        });
+    }
+    catch (error) {
+        result = buildKeyboardCloseResult(KEYBOARD_CLOSE_RESULTS.FAILED, error);
+    }
+    const addOutput = context?.addOutput || printCliOutput;
+    const outputText = result.status === KEYBOARD_CLOSE_RESULTS.FAILED
+        ? `${result.message}. The preceding text entry was unaffected.`
+        : result.message;
+    const scope = isDesignContext(context) ? "design" : "execution";
+    const ids = {
+        ...(scope === "design" ? { sessionId: context?.sessionId } : { runId: context?.runId }),
+        ...(stepContext?.stepId ? { stepId: stepContext.stepId } : {}),
+        ...(Number.isInteger(stepContext?.instructionIndex)
+            ? { instructionIndex: stepContext.instructionIndex }
+            : {}),
+    };
+    const payload = {
+        platform: session.deviceInfo?.platform,
+        toolName: "close_keyboard",
+        callId,
+        status: result.status,
+        failed: result.status === KEYBOARD_CLOSE_RESULTS.FAILED,
+        ...(dismissalPoint ? { dismissalPoint } : {}),
+        ...(result.error ? { error: result.error } : {}),
+    };
+    emitDesktopDebug("tool_call", scope, ids, payload);
+    addOutput({
+        type: "action",
+        text: result.message,
+        eventType: "tool_call",
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        payload,
+    });
+    session.addToTranscript(`[Close keyboard] ${outputText}`);
+    const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
+    const currentScreenshot = await engine.recordScreenshot(screenshotBase64, {
+        runId: context?.runId,
+        sessionId: context?.sessionId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
+    });
+    return {
+        kind: "continue",
+        outputItem: buildFunctionCallOutputItem(callId, buildToolOutputContent({ text: outputText, imageBase64: screenshotBase64 })),
+        currentScreenshot,
+        executedActions: [{ type: "close_keyboard", ...(dismissalPoint || {}) }],
+    };
+}

package/build/src/device/actions.js

@@ -20,3 +20,10 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
     const backend = getDeviceBackend(platform);
     return backend.handleModelAction(deviceId, action, scale, context);
 }
+export async function closeKeyboard(deviceId, options = {}) {
+    const platform = getCurrentPlatform();
+    if (!platform) {
+        throw new Error("No platform set. Call connectToDevice first.");
+    }
+    return getDeviceBackend(platform).closeKeyboard(deviceId, options);
+}

package/build/src/device/android/actions.js

@@ -6,9 +6,25 @@ import { buildMobileKeypressPayload, getAndroidAdbTransport, normalizeMobileKeyp
 import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
 import { execAdb } from "./tools.js";
+import { closeAdbKeyboard, resolveAdbKeyboardVisibility } from "../keyboard.js";
 function adbShell(deviceId, command) {
     return execAdb(["-s", deviceId, "shell", command]);
 }
+async function getKeyboardVisibility(deviceId) {
+    const { stdout: inputMethodDump } = await adbShell(deviceId, "dumpsys input_method");
+    const { stdout: showImeWithHardwareKeyboard } = await adbShell(deviceId, "settings get secure show_ime_with_hard_keyboard");
+    return resolveAdbKeyboardVisibility(inputMethodDump, showImeWithHardwareKeyboard);
+}
+export async function closeKeyboard(deviceId) {
+    // KEYCODE_BACK dismisses the Android IME, but it would navigate if the app
+    // already hid the keyboard. getKeyboardVisibility also filters the hardware
+    // keyboard configuration where mInputShown is true without a software input
+    // view. closeAdbKeyboard invokes Back only for an affirmative result.
+    return closeAdbKeyboard({
+        getVisibility: () => getKeyboardVisibility(deviceId),
+        pressBack: () => adbShell(deviceId, "input keyevent KEYCODE_BACK"),
+    });
+}
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const meta = (payload = {}) => buildActionEventMeta({

package/build/src/device/cloud/actions.js

@@ -6,6 +6,33 @@ import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSe
 import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
+import { closeAppiumKeyboard, closeAppiumKeyboardWithTap } from "../keyboard.js";
+export async function closeKeyboard(_deviceId, { dismissalPoint = null, scale = 1 } = {}) {
+    const session = getActiveSession();
+    if (!session?.client) {
+        throw new Error("No active cloud session");
+    }
+    if (session.platform === "ios") {
+        // XCUITest hideKeyboard was unreliable on the tested AWS real devices, and
+        // Return/newline can submit forms. Trust the model to identify a dismissal
+        // control or inert background, but tap only after confirming the keyboard
+        // is open. With no trusted point, fail safely without touching the screen.
+        const dpr = getDevicePixelRatio();
+        const tap = dismissalPoint
+            ? () => session.client.tap(session.sessionId, Math.round((dismissalPoint.x / scale) / dpr), Math.round((dismissalPoint.y / scale) / dpr))
+            : null;
+        return closeAppiumKeyboardWithTap({
+            isKeyboardShown: () => session.client.isKeyboardShown(session.sessionId),
+            tap,
+        });
+    }
+    // UiAutomator2's native query/hide commands are reliable on cloud Android and
+    // avoid the navigation risk of treating Android Back as a generic fallback.
+    return closeAppiumKeyboard({
+        isKeyboardShown: () => session.client.isKeyboardShown(session.sessionId),
+        hideKeyboard: () => session.client.hideKeyboard(session.sessionId),
+    });
+}
 export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
     const session = getActiveSession();

package/build/src/device/cloud/aws-device-farm/upload-cache.js

@@ -7,7 +7,7 @@ import { reportCloudProgress } from "../progress.js";
 export const DEFAULT_UPLOAD_CACHE_PATH = path.join(os.tmpdir(), "droid-cua", "aws-device-farm-upload-cache.json");
 // Stream the binary through the hash rather than buffering the whole .apk/.ipa
 // (tens–hundreds of MB) in one allocation just to checksum it.
-async function hashFile(filePath) {
+export async function hashFile(filePath) {
     const hash = createHash("sha256");
     for await (const chunk of createReadStream(filePath)) {
         hash.update(chunk);

package/build/src/device/cloud/cloud-appium-client.js

@@ -99,6 +99,21 @@ export class CloudAppiumClient {
             ]
         });
     }
+    async isKeyboardShown(sessionId) {
+        const response = await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: isKeyboardShown",
+            args: [{}]
+        });
+        return response.value === true;
+    }
+    async hideKeyboard(sessionId) {
+        // Currently called by cloud Android only. Cloud iOS deliberately routes to
+        // its guarded model-selected tap in cloud/actions.js.
+        await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: hideKeyboard",
+            args: [{}]
+        });
+    }
     async scroll(sessionId, startX, startY, endX, endY, duration = 500) {
         await this.request("POST", `/session/${sessionId}/actions`, {
             actions: [

package/build/src/device/cloud/connection.js

@@ -4,7 +4,7 @@ import { CloudAppiumClient } from "./cloud-appium-client.js";
 import { reportCloudPhase } from "./progress.js";
 let activeSession = null;
 function buildCloudCapabilities(options) {
-    const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName } = options;
+    const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName, cloudSessionId, artifacts } = options;
     const adapter = getCloudProviderAdapter(providerId);
     if (!adapter) {
         throw new Error(`Cloud provider "${providerId}" is not registered.`);
@@ -20,7 +20,9 @@ function buildCloudCapabilities(options) {
             accessKey: credentials?.accessKey,
             projectName: "droid-cua",
             buildName: buildName || `desktop-${new Date().toISOString().slice(0, 10)}`,
-            sessionName: sessionName || `${deviceName} (${platform})`
+            sessionName: sessionName || `${deviceName} (${platform})`,
+            cloudSessionId,
+            artifacts
         }
     };
 }
@@ -29,11 +31,19 @@ export async function connectToDevice(options, logOptions = {}) {
     if (!options || typeof options !== "object") {
         throw new Error("Cloud device connection requires a provider payload.");
     }
-    const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName } = options;
-    if (!providerId || !platform || !deviceName || !osVersion || !appRef) {
+    const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName, cloudSessionId, artifacts } = options;
+    const isReconnect = providerId === "loadmill-cloud" && Boolean(cloudSessionId);
+    if (!providerId ||
+        (!isReconnect && (!platform || !deviceName || !osVersion)) ||
+        (!appRef && !isReconnect)) {
         throw new Error("Cloud device connection is missing required session details.");
     }
-    if (!credentials?.username || !credentials?.accessKey) {
+    if (providerId === "loadmill-cloud") {
+        if (!credentials?.bearerToken) {
+            throw new Error("Loadmill Cloud connection is missing a Loadmill API token.");
+        }
+    }
+    else if (!credentials?.username || !credentials?.accessKey) {
         throw new Error("Cloud device connection is missing provider credentials.");
     }
     const { adapter, capabilities } = buildCloudCapabilities({
@@ -44,7 +54,9 @@ export async function connectToDevice(options, logOptions = {}) {
         appRef,
         credentials,
         buildName,
-        sessionName
+        sessionName,
+        cloudSessionId,
+        artifacts
     });
     // Optional AbortSignal (Stop/Pause during provisioning). It rides inside the
     // cloud-options object — see the facade in src/device/connection.js — so no