@loadmill/droid-cua 2.19.0 → 2.21.0

package/build/src/device/cloud/connection.js

@@ -1,6 +1,7 @@
 import sharp from "sharp";
 import { getCloudProviderAdapter } from "./registry.js";
 import { CloudAppiumClient } from "./cloud-appium-client.js";
+import { reportCloudPhase } from "./progress.js";
 let activeSession = null;
 function buildCloudCapabilities(options) {
     const { providerId, platform, deviceName, osVersion, appRef, credentials, buildName, sessionName } = options;
@@ -45,19 +46,55 @@ export async function connectToDevice(options, logOptions = {}) {
         buildName,
         sessionName
     });
+    // Optional AbortSignal (Stop/Pause during provisioning). It rides inside the
+    // cloud-options object — see the facade in src/device/connection.js — so no
+    // positional signatures change across the backend boundary.
+    const signal = options.signal;
     const client = new CloudAppiumClient(platform, adapter, credentials);
-    log(`Creating ${adapter.displayName} session on ${deviceName} (${osVersion})...`);
-    const session = await client.createSession(capabilities);
-    activeSession = {
-        client,
-        providerId,
-        platform,
-        deviceName,
-        osVersion,
-        sessionId: session.sessionId
-    };
-    log(`Connected to ${adapter.displayName} device "${deviceName}" (${session.sessionId})`);
-    return session.sessionId;
+    let prepared = false;
+    if (typeof adapter.prepareSession === "function") {
+        log(`Provisioning ${adapter.displayName} device "${deviceName}" (this can take ~1 minute)...`);
+        await adapter.prepareSession(credentials, capabilities, { signal });
+        prepared = true;
+    }
+    // The session reached RUNNING just as a Stop landed: release the freshly
+    // created (billable) session before opening a WebDriver session on it. The
+    // poll itself aborts mid-PREPARING via the signal above; this covers only the
+    // narrow window after RUNNING but before createSession.
+    if (signal?.aborted && prepared && typeof adapter.releaseSession === "function") {
+        await adapter.releaseSession(credentials, { waitForCompletion: false }).catch(() => { });
+        throw signal.reason ?? new Error("Provisioning cancelled.");
+    }
+    try {
+        reportCloudPhase("create WebDriver session");
+        log(`Creating ${adapter.displayName} session on ${deviceName} (${osVersion})...`);
+        const session = await client.createSession(capabilities);
+        activeSession = {
+            client,
+            providerId,
+            platform,
+            deviceName,
+            osVersion,
+            sessionId: session.sessionId
+        };
+        log(`Connected to ${adapter.displayName} device "${deviceName}" (${session.sessionId})`);
+        return session.sessionId;
+    }
+    catch (error) {
+        // prepareSession already created a billable remote-access session, but
+        // activeSession was never set — so disconnect() can't tear it down. Release
+        // it here (best-effort, snappy) so the failed connect doesn't leak a session
+        // that bills until the provider's idle timeout. Don't mask the real error.
+        if (prepared && typeof adapter.releaseSession === "function") {
+            try {
+                await adapter.releaseSession(credentials, { waitForCompletion: false });
+            }
+            catch (releaseError) {
+                log(`Failed to release ${adapter.displayName} session after a connection error: ${releaseError.message}`);
+            }
+        }
+        throw error;
+    }
 }
 export async function getDeviceInfo(sessionId) {
     if (!activeSession || activeSession.sessionId !== sessionId) {
@@ -114,14 +151,34 @@ export function setActiveSessionForTests(session) {
 export function clearActiveSessionForTests() {
     activeSession = null;
 }
-export async function disconnect() {
+export async function disconnect(options = {}) {
     if (!activeSession) {
         return;
     }
+    const session = activeSession;
+    let teardownError = null;
+    try {
+        reportCloudPhase("delete WebDriver session");
+        await session.client.deleteSession(session.sessionId);
+    }
+    catch (error) {
+        teardownError = error;
+    }
     try {
-        await activeSession.client.deleteSession(activeSession.sessionId);
+        const provider = session.client.provider;
+        if (typeof provider?.releaseSession === "function") {
+            await provider.releaseSession(session.client.creds, {
+                waitForCompletion: options.waitForCompletion !== false,
+            });
+        }
+    }
+    catch (error) {
+        teardownError ??= error;
     }
     finally {
         activeSession = null;
     }
+    if (teardownError) {
+        throw teardownError;
+    }
 }

package/build/src/device/cloud/progress.js

@@ -0,0 +1,24 @@
+/**
+ * Volatile progress reporting for long-running cloud-provider operations
+ * (upload processing, device provisioning). Consumers install a single
+ * listener that receives { kind, message } events:
+ *   - "detail": volatile status, safe to overwrite (poll results etc.)
+ *   - "phase":  a named sub-step of the current operation has started; the
+ *               previous phase (if any) just ended
+ *   - "note":   persistent information worth keeping after the line clears
+ *               (e.g. billed device-minutes)
+ * With no listener installed, reports are dropped.
+ */
+let listener = null;
+export function setCloudProgressListener(fn) {
+    listener = fn;
+}
+export function reportCloudProgress(message) {
+    listener?.({ kind: "detail", message });
+}
+export function reportCloudPhase(name) {
+    listener?.({ kind: "phase", message: name });
+}
+export function reportCloudNote(message) {
+    listener?.({ kind: "note", message });
+}

package/build/src/device/cloud/registry.js

@@ -1,6 +1,7 @@
 import { browserStackAdapter } from "./browserstack/adapter.js";
 import { lambdaTestAdapter } from "./lambdatest/adapter.js";
-const availableAdapters = [browserStackAdapter, lambdaTestAdapter];
+import { awsDeviceFarmAdapter } from "./aws-device-farm/adapter.js";
+const availableAdapters = [browserStackAdapter, lambdaTestAdapter, awsDeviceFarmAdapter];
 export function listCloudProviderAdapters() {
     return availableAdapters;
 }

package/build/src/device/connection.js

@@ -61,10 +61,11 @@ export async function getScreenshotAsBase64(deviceId, deviceInfo) {
 export { getCurrentPlatform } from "./factory.js";
 /**
  * Disconnect from the device
+ * @param {object} options - e.g. { waitForCompletion } for cloud backends; local backends ignore it
  */
-export async function disconnect() {
+export async function disconnect(options = {}) {
     if (currentBackend?.disconnect) {
-        await currentBackend.disconnect();
+        await currentBackend.disconnect(options);
     }
     currentBackend = null;
     setCurrentSource("local");

package/build/src/device/loadmill.js

@@ -2,12 +2,22 @@
  * Loadmill instruction handling for script execution
  */
 import { executeLoadmillCommand } from "../integrations/loadmill/index.js";
+import { resolveEffectiveHelperModel } from "../core/runtime-config.js";
 import { printCliOutput } from "../utils/console-output.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 function getLoadmillSiteBaseUrl(loadmillClient = null) {
     const rawBaseUrl = loadmillClient?.getBaseUrl?.() || "https://app.loadmill.com/api";
     return rawBaseUrl.replace(/\/api\/?$/, "");
 }
+function resolveOpenAiLoadmillHelperModel(runtimeConfig = null) {
+    if (runtimeConfig?.llmProviderMode !== "openai") {
+        return null;
+    }
+    return resolveEffectiveHelperModel({
+        llmProviderMode: runtimeConfig.llmProviderMode,
+        cuaModel: runtimeConfig.cuaModel,
+    });
+}
 /**
  * Execute a Loadmill instruction and handle the result
  * @param {string} command - The Loadmill command to execute
@@ -35,6 +45,7 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
     const result = await executeLoadmillCommand(command, {
         loadmillClient,
         openAiClient,
+        helperModel: resolveOpenAiLoadmillHelperModel(context?.runtimeConfig),
         futureInstructions: context?.futureInstructions || [],
         onProgress: ({ message, runId }) => {
             const payload = runId

package/build/src/integrations/loadmill/executor.js

@@ -57,13 +57,13 @@ ${futureScript}`
         }
     ];
 }
-async function selectRelevantParameterNames({ openAiClient, command, flowName, futureInstructions, parameters, }) {
+async function selectRelevantParameterNames({ openAiClient, helperModel, command, flowName, futureInstructions, parameters, }) {
     if (!openAiClient || !Array.isArray(parameters) || parameters.length === 0) {
         return [];
     }
     try {
         const response = await openAiClient.chat.completions.create({
-            model: LOADMILL_MODEL_ALIASES.DEFAULT_AGENT_MODEL,
+            model: helperModel || LOADMILL_MODEL_ALIASES.DEFAULT_AGENT_MODEL,
             messages: buildParameterSelectionMessages({
                 command,
                 flowName,
@@ -98,7 +98,7 @@ function findSelectedSuiteFlowRun(finalResult, selectedFlow) {
     }
     return flowRuns.find((flowRun) => flowRun?.testSuiteFlowId === selectedFlow?.id) || null;
 }
-async function buildRuntimeParameterContext({ loadmillClient, finalResult, selectedFlow, command, flowName, futureInstructions, openAiClient, }) {
+async function buildRuntimeParameterContext({ loadmillClient, finalResult, selectedFlow, command, flowName, futureInstructions, openAiClient, helperModel, }) {
     const selectedSuiteFlowRun = findSelectedSuiteFlowRun(finalResult, selectedFlow);
     if (!selectedSuiteFlowRun?.id || typeof loadmillClient.getTestSuiteFlowRun !== "function") {
         return null;
@@ -112,6 +112,7 @@ async function buildRuntimeParameterContext({ loadmillClient, finalResult, selec
             flowName,
             futureInstructions,
             parameters,
+            helperModel,
         });
         const candidates = selectLoadmillParameterCandidates(parameters, selectedParameterNames);
         const note = renderLoadmillParameterContextNote(candidates);
@@ -198,7 +199,7 @@ async function pollForCompletion(runId, loadmillClient, options = {}) {
  * @returns {Promise<{success: boolean, error?: string, result?: Object}>}
  */
 export async function executeLoadmillCommand(userInput, options = {}) {
-    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, futureInstructions = [], pollTimeoutMs, pollIntervalMs, now, sleepFn, } = options;
+    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, helperModel = null, futureInstructions = [], pollTimeoutMs, pollIntervalMs, now, sleepFn, } = options;
     // Check for API token
     if (!loadmillClient.getApiToken()) {
         return {
@@ -215,7 +216,10 @@ export async function executeLoadmillCommand(userInput, options = {}) {
     try {
         // Step 1: Interpret the command
         onProgress({ step: "interpreting", message: "Interpreting command..." });
-        const interpreted = await interpretLoadmillCommand(userInput, { client: openAiClient });
+        const interpreted = await interpretLoadmillCommand(userInput, {
+            client: openAiClient,
+            ...(helperModel ? { model: helperModel } : {}),
+        });
         logger.debug("Loadmill command interpreted", interpreted);
         // Step 2: Search for flows
         onProgress({ step: "searching", message: `Searching for flows matching "${interpreted.searchQuery}"...` });
@@ -229,7 +233,10 @@ export async function executeLoadmillCommand(userInput, options = {}) {
         }
         // Step 3: Select best match
         onProgress({ step: "selecting", message: `Found ${flows.length} flow(s). Selecting best match...` });
-        const { selectedFlow, confidence } = await selectBestFlow(flows, interpreted.searchQuery, { client: openAiClient });
+        const { selectedFlow, confidence } = await selectBestFlow(flows, interpreted.searchQuery, {
+            client: openAiClient,
+            ...(helperModel ? { model: helperModel } : {}),
+        });
         if (!selectedFlow) {
             return {
                 success: false,
@@ -272,6 +279,7 @@ export async function executeLoadmillCommand(userInput, options = {}) {
                 flowName,
                 futureInstructions,
                 openAiClient,
+                helperModel,
             })
             : null;
         return {

package/build/src/integrations/loadmill/interpreter.js

@@ -8,9 +8,9 @@ import { buildLoadmillCommandInterpretationMessages, buildLoadmillFlowSelectionM
  * @param {string} userInput - Natural language command
  * @returns {Promise<{searchQuery: string, parameters: Object}>}
  */
-export async function interpretLoadmillCommand(userInput, { client }) {
+export async function interpretLoadmillCommand(userInput, { client, model = LOADMILL_MODEL_ALIASES.FAST }) {
     const response = await client.chat.completions.create({
-        model: LOADMILL_MODEL_ALIASES.FAST,
+        model,
         messages: buildLoadmillCommandInterpretationMessages(userInput),
         response_format: { type: "json_object" }
     });
@@ -27,7 +27,7 @@ export async function interpretLoadmillCommand(userInput, { client }) {
  * @param {string} originalQuery - Original user query
  * @returns {Promise<{selectedFlow: Object|null, confidence: number}>}
  */
-export async function selectBestFlow(flows, originalQuery, { client }) {
+export async function selectBestFlow(flows, originalQuery, { client, model = LOADMILL_MODEL_ALIASES.FAST }) {
     // Ensure flows is an array
     if (!flows || !Array.isArray(flows) || flows.length === 0) {
         return { selectedFlow: null, confidence: 0 };
@@ -42,7 +42,7 @@ export async function selectBestFlow(flows, originalQuery, { client }) {
         return `${i + 1}. ID: ${f.id}, Name: "${name}"${suite ? `, Suite: "${suite}"` : ""}`;
     }).join("\n");
     const response = await client.chat.completions.create({
-        model: LOADMILL_MODEL_ALIASES.FAST,
+        model,
         messages: buildLoadmillFlowSelectionMessages(originalQuery, flowList),
         response_format: { type: "json_object" }
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.19.0",
+  "version": "2.21.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -22,6 +22,8 @@
     "bench:report:web": "node scripts/benchmark-report-web.js",
     "execution:profile": "node scripts/execution-profile.js",
     "test": "node --test unit-tests/*.test.js",
+    "test:aws-device-farm:android": "node integration-tests/aws-device-farm-android.test.js",
+    "test:aws-device-farm:ios": "node integration-tests/aws-device-farm-ios.test.js",
     "build": "tsc",
     "clean": "rm -rf build",
     "typecheck": "tsc --noEmit",
@@ -58,6 +60,7 @@
   "author": "Loadmill",
   "license": "UNLICENSED",
   "dependencies": {
+    "@aws-sdk/client-device-farm": "^3.1065.0",
     "dotenv": "^16.4.7",
     "ink": "^4.4.1",
     "ink-spinner": "^5.0.0",