@loadmill/droid-cua 2.14.0 → 2.15.1

package/README.md

@@ -142,6 +142,9 @@ droid-cua
 # Headless Android run
 droid-cua --avd adb:emulator-5554 --instructions tests/login.dcua
 
+# Headless Android folder run
+droid-cua --avd adb:emulator-5554 --instructions tests
+
 # Headless iOS simulator run
 droid-cua --platform ios --avd "iPhone 16" --instructions tests/login.dcua
 

package/build/index.js

@@ -3,14 +3,12 @@ import dotenv from "dotenv";
 import path from "path";
 import { mkdir } from "fs/promises";
 import { connectToDevice, disconnect as disconnectDevice, getDeviceInfo } from "./src/device/connection.js";
-import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "./src/browser/connection.js";
-import { handleModelAction as handleBrowserModelAction } from "./src/browser/actions.js";
+import { connectToBrowser, getBrowserInfo, } from "./src/browser/connection.js";
 import { normalizeBrowserInfoForSession } from "./src/browser/normalize.js";
 import { Session } from "./src/core/session.js";
 import { ExecutionEngine } from "./src/core/execution-engine.js";
-import { buildBaseSystemPrompt, buildExecutionModePrompt } from "./src/core/prompts.js";
+import { buildBaseSystemPrompt } from "./src/core/prompts.js";
 import { startInkShell } from "./src/cli/ink-shell.js";
-import { ExecutionMode } from "./src/modes/execution-mode.js";
 import { createOpenAiExecutor } from "./src/device/openai.js";
 import { logger } from "./src/utils/logger.js";
 import { selectDevice } from "./src/cli/device-selector.js";
@@ -18,10 +16,10 @@ import { resolveExecutionAppContext } from "./src/core/app-context.js";
 import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
 import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
-import { printCliOutput } from "./src/utils/console-output.js";
-import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
 import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
-import { loadHeadlessInstructions, resolveHeadlessInstructionsPath } from "./src/cli/headless-instructions.js";
+import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
+import { listHeadlessInstructionFiles, resolveHeadlessInstructionsTarget } from "./src/cli/headless-instruction-target.js";
+import { runHeadlessInstructionRuns } from "./src/cli/headless-runner.js";
 import { resolveLambdaTestCredentials, uploadLambdaTestCliApp } from "./src/cli/lambdatest-app-upload.js";
 import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
 import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
@@ -109,13 +107,30 @@ async function main() {
         throw new Error("--device-source lambdatest is only supported with --instructions headless execution.");
     }
     let preloadedHeadlessConfig = null;
-    let preloadedHeadlessInstructions = null;
-    let resolvedInstructionsFile = instructionsFile;
+    let preloadedHeadlessTarget = null;
+    let headlessInstructionRuns = [];
     let preloadedAppContext = null;
     if (isHeadlessInstructionsRun) {
         preloadedHeadlessConfig = await resolveHeadlessExecutionConfig(args);
-        resolvedInstructionsFile = await resolveHeadlessInstructionsPath(instructionsFile);
-        preloadedHeadlessInstructions = await loadHeadlessInstructions(resolvedInstructionsFile);
+        preloadedHeadlessTarget = await resolveHeadlessInstructionsTarget(instructionsFile);
+        if (preloadedHeadlessTarget.type === "directory") {
+            const files = await listHeadlessInstructionFiles(preloadedHeadlessTarget.path);
+            if (files.length === 0) {
+                throw new Error(`No .dcua tests found in instructions directory ${preloadedHeadlessTarget.path}.`);
+            }
+            headlessInstructionRuns = files.map((file) => ({
+                testPath: file.path,
+                testName: file.filename,
+                instructions: null
+            }));
+        }
+        else {
+            headlessInstructionRuns = [{
+                    testPath: preloadedHeadlessTarget.path,
+                    testName: path.basename(preloadedHeadlessTarget.path),
+                    instructions: await loadHeadlessInstructions(preloadedHeadlessTarget.path)
+                }];
+        }
         preloadedAppContext = await resolveExecutionAppContext({
             enabled: preloadedHeadlessConfig.appContextEnabled,
             contextPath: preloadedHeadlessConfig.appContextPath,
@@ -198,25 +213,9 @@ async function main() {
     session.setSystemPrompt(initialSystemText);
     // If --instructions provided, run in headless mode
     if (instructionsFile) {
-        const runId = `run-${Date.now()}`;
-        const testName = path.basename(resolvedInstructionsFile);
-        let instructions = [];
-        let executionMode = null;
         let exitCode = null;
         try {
             const headlessConfig = preloadedHeadlessConfig;
-            await headlessDebug.startExecutionSession(runId, {
-                testName,
-                platform: deviceInfo.platform,
-                deviceName: deviceInfo.device_name,
-                aimAssistMode: headlessConfig.aimAssistMode
-            });
-            const currentLogFilePath = await headlessDebug.getCurrentLogFilePath();
-            if (currentLogFilePath) {
-                console.log(`Debug logging enabled: ${currentLogFilePath}`);
-            }
-            console.log(`\nRunning test from: ${resolvedInstructionsFile}\n`);
-            instructions = preloadedHeadlessInstructions;
             const appContext = preloadedAppContext;
             const appContextText = appContext.text;
             const runtimeContext = createCliRuntimeContext({
@@ -232,130 +231,21 @@ async function main() {
                     message: appContext.message,
                 },
             });
-            console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
-            if (appContext.status === "loaded" || appContext.status === "empty") {
-                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
-                    source: "cli_context_flag",
-                    contextPath: appContext.contextPath,
-                    status: appContext.status,
-                    text: appContextText
-                });
-                if (appContextText) {
-                    console.log(`Using app context from: ${appContext.contextPath}`);
-                }
-            }
-            else if (appContext.status === "failed") {
-                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
-                    source: "cli_context_flag",
-                    contextPath: appContext.contextPath,
-                    status: "failed",
-                    message: appContext.message
-                });
-                console.warn(`Warning: could not load app context from ${appContext.contextPath}. Running without it.`);
-            }
-            else {
-                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
-                    source: "cli_context_flag",
-                    contextPath: appContext.contextPath,
-                    status: appContext.status
-                });
-            }
-            const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { aimAssistMode: headlessConfig.aimAssistMode });
-            session.setSystemPrompt(executionPrompt);
-            const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
-                runId,
+            const result = await runHeadlessInstructionRuns({
+                runs: headlessInstructionRuns,
+                instructionsTarget: preloadedHeadlessTarget,
+                session,
+                deviceInfo,
+                headlessConfig,
+                appContext,
+                runtimeContext,
+                headlessDebug,
                 recordScreenshots,
-                screenshotDir
-            });
-            const executionEngineOptions = {
-                toolCapabilities: {
-                    assert: true,
-                    navigate: isBrowserTarget,
-                    loadmill: true,
-                },
-                baseSystemPrompt: executionPrompt,
-                isHeadlessMode: true,
-            };
-            const engineOptions = screenshotRecorder
-                ? {
-                    recordScreenshots: true,
-                    screenshotRecorder,
-                    stepDelayMs: runtimeContext.config.stepDelayMs,
-                    sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
-                        ...requestArgs,
-                        aimAssistMode: headlessConfig.aimAssistMode,
-                    }),
-                    reviseTestScript: runtimeContext.openAi.reviseTestScript,
-                    ...executionEngineOptions,
-                }
-                : {
-                    recordScreenshots,
-                    screenshotDir,
-                    stepDelayMs: runtimeContext.config.stepDelayMs,
-                    sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
-                        ...requestArgs,
-                        aimAssistMode: headlessConfig.aimAssistMode,
-                    }),
-                    reviseTestScript: runtimeContext.openAi.reviseTestScript,
-                    ...executionEngineOptions,
-                };
-            if (isBrowserTarget) {
-                engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
-                engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
-                engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
-            }
-            else if (isLambdaTestTarget) {
-                engineOptions.connectToDevice = async () => {
-                    throw new Error("LambdaTest cloud session disconnected; automatic CLI reconnect is not supported yet.");
-                };
-            }
-            const engine = new ExecutionEngine(session, engineOptions);
-            executionMode = new ExecutionMode(session, engine, instructions, true);
-            const result = await executionMode.execute({
-                runId,
-                addOutput: printCliOutput,
-                openAiClient: runtimeContext.openAiClient,
-                loadmillClient: runtimeContext.loadmillClient,
-                runtimeCredentials: runtimeContext.credentials,
-            });
-            const stats = executionMode.stats || {};
-            const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
-            await headlessDebug.endExecutionSession(runId, {
-                success: Boolean(result.success),
-                error: result.error ?? null,
-                durationMs,
-                instructionsTotal: instructions.length,
-                instructionsCompleted: stats.instructionsCompleted ?? 0,
-                actionsTotal: stats.actionCount ?? 0,
-                assertionsPassed: stats.assertionsPassed ?? 0,
-                assertionsFailed: stats.assertionsFailed ?? 0,
-                retries: stats.retryCount ?? 0
+                screenshotDir,
+                isBrowserTarget,
+                isLambdaTestTarget
             });
-            if (!result.success) {
-                console.error(`\nTest failed: ${result.error}`);
-            }
-            exitCode = result.success ? 0 : 1;
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : "Failed to start execution.";
-            const stats = executionMode?.stats || {};
-            const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
-            await headlessDebug.endExecutionSession(runId, {
-                success: false,
-                error: message,
-                ...(stats.startTime
-                    ? {
-                        durationMs,
-                        instructionsTotal: instructions.length,
-                        instructionsCompleted: stats.instructionsCompleted ?? 0,
-                        actionsTotal: stats.actionCount ?? 0,
-                        assertionsPassed: stats.assertionsPassed ?? 0,
-                        assertionsFailed: stats.assertionsFailed ?? 0,
-                        retries: stats.retryCount ?? 0
-                    }
-                    : { reason: "start_failed" })
-            });
-            throw error;
+            exitCode = result.exitCode;
         }
         finally {
             if (lambdaTestSessionConnected) {

package/build/src/cli/headless-instruction-target.js

@@ -0,0 +1,64 @@
+import path from "path";
+import { readdir, stat } from "fs/promises";
+import { resolveHeadlessInstructionsPath } from "./headless-instructions.js";
+const testNameCollator = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" });
+const filenameCollator = new Intl.Collator(undefined, { sensitivity: "base" });
+function compareInstructionFiles(left, right) {
+    const byName = testNameCollator.compare(left.name, right.name);
+    if (byName !== 0) {
+        return byName;
+    }
+    const byModified = right.modified.localeCompare(left.modified);
+    if (byModified !== 0) {
+        return byModified;
+    }
+    return filenameCollator.compare(left.filename, right.filename);
+}
+/**
+ * Resolve a CLI instructions target as either a single .dcua file or a directory
+ * containing .dcua files.
+ *
+ * @param {string} instructionsPath
+ * @returns {Promise<{type: 'file'|'directory', path: string}>}
+ */
+export async function resolveHeadlessInstructionsTarget(instructionsPath) {
+    const resolvedPath = await resolveHeadlessInstructionsPath(instructionsPath);
+    try {
+        const pathStat = await stat(resolvedPath);
+        if (pathStat.isDirectory()) {
+            return { type: "directory", path: resolvedPath };
+        }
+    }
+    catch {
+        // Missing paths stay as file targets so the existing read error remains clear.
+    }
+    return { type: "file", path: resolvedPath };
+}
+/**
+ * List runnable .dcua files from a headless instructions directory.
+ *
+ * @param {string} instructionsDirectory
+ * @returns {Promise<Array<{name: string, filename: string, path: string, modified: string}>>}
+ */
+export async function listHeadlessInstructionFiles(instructionsDirectory) {
+    let entries;
+    try {
+        entries = await readdir(instructionsDirectory);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : "Unknown directory read error.";
+        throw new Error(`Could not read instructions directory ${instructionsDirectory}: ${message}`);
+    }
+    const dcuaFiles = entries.filter((entry) => entry.toLowerCase().endsWith(".dcua"));
+    const tests = await Promise.all(dcuaFiles.map(async (filename) => {
+        const fullPath = path.join(instructionsDirectory, filename);
+        const fileStat = await stat(fullPath);
+        return {
+            name: filename.replace(/\.dcua$/i, ""),
+            filename,
+            path: fullPath,
+            modified: fileStat.mtime.toISOString()
+        };
+    }));
+    return tests.sort(compareInstructionFiles);
+}

package/build/src/cli/headless-runner.js

@@ -0,0 +1,245 @@
+import path from "path";
+import { mkdir } from "fs/promises";
+import { getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "../browser/connection.js";
+import { handleModelAction as handleBrowserModelAction } from "../browser/actions.js";
+import { ExecutionEngine } from "../core/execution-engine.js";
+import { buildExecutionModePrompt } from "../core/prompts.js";
+import { ExecutionMode } from "../modes/execution-mode.js";
+import { printCliOutput } from "../utils/console-output.js";
+import { emitDesktopDebug } from "../utils/desktop-debug.js";
+import { loadHeadlessInstructions } from "./headless-instructions.js";
+function sanitizePathSegment(value) {
+    return String(value || "test")
+        .replace(/\.dcua$/i, "")
+        .trim()
+        .replace(/[^a-zA-Z0-9._-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^[-_.]+|[-_.]+$/g, "") || "test";
+}
+function createHeadlessRunId(index = 0) {
+    const suffix = index > 0 ? `-${index + 1}` : "";
+    return `run-${Date.now()}${suffix}`;
+}
+function getRecordScreenshotDirForTest(baseScreenshotDir, testName, isBatchRun) {
+    if (!baseScreenshotDir || !isBatchRun) {
+        return baseScreenshotDir;
+    }
+    return path.join(baseScreenshotDir, sanitizePathSegment(testName));
+}
+function emitCliAppContextDebug(runId, appContext, appContextText) {
+    if (appContext.status === "loaded" || appContext.status === "empty") {
+        emitDesktopDebug("app_context.loaded", "execution", { runId }, {
+            source: "cli_context_flag",
+            contextPath: appContext.contextPath,
+            status: appContext.status,
+            text: appContextText
+        });
+        if (appContextText) {
+            console.log(`Using app context from: ${appContext.contextPath}`);
+        }
+    }
+    else if (appContext.status === "failed") {
+        emitDesktopDebug("app_context.loaded", "execution", { runId }, {
+            source: "cli_context_flag",
+            contextPath: appContext.contextPath,
+            status: "failed",
+            message: appContext.message
+        });
+        console.warn(`Warning: could not load app context from ${appContext.contextPath}. Running without it.`);
+    }
+    else {
+        emitDesktopDebug("app_context.loaded", "execution", { runId }, {
+            source: "cli_context_flag",
+            contextPath: appContext.contextPath,
+            status: appContext.status
+        });
+    }
+}
+function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionPrompt, screenshotRecorder, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
+    const executionEngineOptions = {
+        toolCapabilities: {
+            assert: true,
+            navigate: isBrowserTarget,
+            loadmill: true,
+        },
+        baseSystemPrompt: executionPrompt,
+        isHeadlessMode: true,
+    };
+    const engineOptions = screenshotRecorder
+        ? {
+            recordScreenshots: true,
+            screenshotRecorder,
+            stepDelayMs: runtimeContext.config.stepDelayMs,
+            sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
+                ...requestArgs,
+                aimAssistMode: headlessConfig.aimAssistMode,
+            }),
+            reviseTestScript: runtimeContext.openAi.reviseTestScript,
+            ...executionEngineOptions,
+        }
+        : {
+            recordScreenshots,
+            screenshotDir,
+            stepDelayMs: runtimeContext.config.stepDelayMs,
+            sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
+                ...requestArgs,
+                aimAssistMode: headlessConfig.aimAssistMode,
+            }),
+            reviseTestScript: runtimeContext.openAi.reviseTestScript,
+            ...executionEngineOptions,
+        };
+    if (isBrowserTarget) {
+        engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
+        engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
+        engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
+    }
+    else if (isLambdaTestTarget) {
+        engineOptions.connectToDevice = async () => {
+            throw new Error("LambdaTest cloud session disconnected; automatic CLI reconnect is not supported yet.");
+        };
+    }
+    return engineOptions;
+}
+async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
+    let executionMode = null;
+    try {
+        await headlessDebug.startExecutionSession(runId, {
+            testName,
+            platform: deviceInfo.platform,
+            deviceName: deviceInfo.device_name,
+            aimAssistMode: headlessConfig.aimAssistMode
+        });
+        const currentLogFilePath = await headlessDebug.getCurrentLogFilePath();
+        if (currentLogFilePath) {
+            console.log(`Debug logging enabled: ${currentLogFilePath}`);
+        }
+        console.log(`\nRunning test from: ${testPath}\n`);
+        console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
+        const appContextText = appContext.text;
+        emitCliAppContextDebug(runId, appContext, appContextText);
+        session.updateResponseId(undefined);
+        session.clearMessages();
+        const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { aimAssistMode: headlessConfig.aimAssistMode });
+        session.setSystemPrompt(executionPrompt);
+        const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
+            runId,
+            recordScreenshots,
+            screenshotDir
+        });
+        const engineOptions = buildHeadlessEngineOptions({
+            runtimeContext,
+            headlessConfig,
+            executionPrompt,
+            screenshotRecorder,
+            recordScreenshots,
+            screenshotDir,
+            isBrowserTarget,
+            isLambdaTestTarget
+        });
+        const engine = new ExecutionEngine(session, engineOptions);
+        executionMode = new ExecutionMode(session, engine, instructions, true);
+        const result = await executionMode.execute({
+            runId,
+            addOutput: printCliOutput,
+            openAiClient: runtimeContext.openAiClient,
+            loadmillClient: runtimeContext.loadmillClient,
+            runtimeCredentials: runtimeContext.credentials,
+        });
+        const stats = executionMode.stats || {};
+        const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
+        await headlessDebug.endExecutionSession(runId, {
+            success: Boolean(result.success),
+            error: result.error ?? null,
+            durationMs,
+            instructionsTotal: instructions.length,
+            instructionsCompleted: stats.instructionsCompleted ?? 0,
+            actionsTotal: stats.actionCount ?? 0,
+            assertionsPassed: stats.assertionsPassed ?? 0,
+            assertionsFailed: stats.assertionsFailed ?? 0,
+            retries: stats.retryCount ?? 0
+        });
+        if (!result.success) {
+            console.error(`\nTest failed: ${result.error}`);
+        }
+        return {
+            success: Boolean(result.success),
+            error: result.error ?? null
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : "Failed to start execution.";
+        const stats = executionMode?.stats || {};
+        const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
+        await headlessDebug.endExecutionSession(runId, {
+            success: false,
+            error: message,
+            ...(stats.startTime
+                ? {
+                    durationMs,
+                    instructionsTotal: instructions.length,
+                    instructionsCompleted: stats.instructionsCompleted ?? 0,
+                    actionsTotal: stats.actionCount ?? 0,
+                    assertionsPassed: stats.assertionsPassed ?? 0,
+                    assertionsFailed: stats.assertionsFailed ?? 0,
+                    retries: stats.retryCount ?? 0
+                }
+                : { reason: "start_failed" })
+        });
+        throw error;
+    }
+}
+export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
+    const isBatchRun = instructionsTarget?.type === "directory";
+    if (isBatchRun) {
+        console.log(`\nRunning ${runs.length} tests from: ${instructionsTarget.path}\n`);
+    }
+    let failures = 0;
+    for (const [index, run] of runs.entries()) {
+        const runId = createHeadlessRunId(index);
+        const testScreenshotDir = getRecordScreenshotDirForTest(screenshotDir, run.testName, isBatchRun);
+        if (isBatchRun) {
+            console.log(`\n[${index + 1}/${runs.length}] ${run.testName}`);
+        }
+        try {
+            const instructions = run.instructions ?? await loadHeadlessInstructions(run.testPath);
+            if (recordScreenshots && testScreenshotDir) {
+                await mkdir(testScreenshotDir, { recursive: true });
+            }
+            const result = await runHeadlessTest({
+                runId,
+                testPath: run.testPath,
+                testName: run.testName,
+                instructions,
+                session,
+                deviceInfo,
+                headlessConfig,
+                appContext,
+                runtimeContext,
+                headlessDebug,
+                recordScreenshots,
+                screenshotDir: testScreenshotDir,
+                isBrowserTarget,
+                isLambdaTestTarget
+            });
+            if (!result.success) {
+                failures += 1;
+            }
+        }
+        catch (error) {
+            failures += 1;
+            if (!isBatchRun) {
+                throw error;
+            }
+            const message = error instanceof Error ? error.message : String(error);
+            console.error(`\nTest failed: ${run.testName}: ${message}`);
+        }
+    }
+    if (isBatchRun) {
+        const passed = runs.length - failures;
+        console.log(`\nFolder run complete: ${passed}/${runs.length} passed`);
+    }
+    return {
+        exitCode: failures === 0 ? 0 : 1,
+        failures
+    };
+}

package/build/src/commands/loadmill.js

@@ -21,12 +21,12 @@ export async function handleLoadmill(args, session, context) {
         addOutput({ type: 'info', text: '  /loadmill <command>          Execute a Loadmill flow' });
         addOutput({ type: 'info', text: '' });
         addOutput({ type: 'info', text: 'Examples:' });
-        addOutput({ type: 'info', text: '  /loadmill search for login flow' });
+        addOutput({ type: 'info', text: '  /loadmill login flow' });
         addOutput({ type: 'info', text: '  /loadmill run checkout flow with user=test123' });
         addOutput({ type: 'info', text: '  /loadmill run payment test with amount=100' });
         addOutput({ type: 'info', text: '' });
         addOutput({ type: 'info', text: 'In test scripts (.dcua files):' });
-        addOutput({ type: 'info', text: '  loadmill: run user authentication flow' });
+        addOutput({ type: 'info', text: '  run loadmill user authentication flow' });
         addOutput({ type: 'info', text: '' });
         addOutput({ type: 'info', text: 'Configuration:' });
         addOutput({ type: 'info', text: '  Set LOADMILL_API_TOKEN in your .env file' });
@@ -68,25 +68,9 @@ export async function handleLoadmill(args, session, context) {
     }
     // Display results
     if (result.success) {
-        if (result.action === "search") {
-            addOutput({ type: 'success', text: `Found ${result.result.flows.length} flow(s):` });
-            result.result.flows.forEach((flow, i) => {
-                const name = flow.description || flow.name || "Unknown";
-                const suite = flow.testSuiteDescription ? ` (Suite: ${flow.testSuiteDescription})` : '';
-                addOutput({ type: 'info', text: `  ${i + 1}. ${name}${suite}` });
-                addOutput({ type: 'info', text: `     ID: ${flow.id}` });
-            });
-            if (result.result.selectedFlow) {
-                const selectedName = result.result.selectedFlow.description || result.result.selectedFlow.name || "Unknown";
-                addOutput({ type: 'info', text: '' });
-                addOutput({ type: 'info', text: `Best match: "${selectedName}" (confidence: ${(result.result.confidence * 100).toFixed(0)}%)` });
-            }
-        }
-        else {
-            addOutput({ type: 'success', text: `Flow "${result.flowName}" passed` });
-            if (result.runId) {
-                addOutput({ type: 'info', text: `Run ID: ${result.runId}` });
-            }
+        addOutput({ type: 'success', text: `Flow "${result.flowName}" passed` });
+        if (result.runId) {
+            addOutput({ type: 'info', text: `Run ID: ${result.runId}` });
         }
     }
     else {

package/build/src/device/loadmill.js

@@ -8,34 +8,6 @@ function getLoadmillSiteBaseUrl(loadmillClient = null) {
     const rawBaseUrl = loadmillClient?.getBaseUrl?.() || "https://app.loadmill.com/api";
     return rawBaseUrl.replace(/\/api\/?$/, "");
 }
-/**
- * Check if an instruction is a Loadmill command
- * @param {string} userInput - The instruction text
- * @returns {boolean}
- */
-export function isLoadmillInstruction(userInput) {
-    const trimmed = userInput.trim();
-    const lower = trimmed.toLowerCase();
-    return lower.startsWith("loadmill:") || lower.startsWith("loadmill ");
-}
-/**
- * Extract the Loadmill command from an instruction
- * @param {string} userInput - The instruction text
- * @returns {string} - The extracted command
- */
-export function extractLoadmillCommand(userInput) {
-    const trimmed = userInput.trim();
-    const lower = trimmed.toLowerCase();
-    // Handle "loadmill:" or "Loadmill:"
-    if (lower.startsWith("loadmill:")) {
-        return trimmed.substring("loadmill:".length).trim();
-    }
-    // Handle "loadmill " or "Loadmill "
-    if (lower.startsWith("loadmill ")) {
-        return trimmed.substring("loadmill".length).trim();
-    }
-    return trimmed;
-}
 /**
  * Execute a Loadmill instruction and handle the result
  * @param {string} command - The Loadmill command to execute
@@ -163,57 +135,48 @@ export function handleLoadmillSuccess(command, result, context, stepContext = nu
     const payload = result.runId
         ? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
         : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
-    if (result.action === "search") {
-        addOutput({ type: 'success', text: `[Loadmill] Found ${result.result.flows.length} flow(s)`, eventType: 'system_message', payload, ...meta });
-        result.result.flows.forEach((flow, i) => {
-            const name = flow.description || flow.name || "Unknown";
-            addOutput({ type: 'info', text: `  ${i + 1}. ${name} (ID: ${flow.id})`, eventType: 'system_message', payload, ...meta });
+    addOutput({ type: 'success', text: `[Loadmill] Flow "${result.flowName}" passed`, eventType: 'system_message', payload, ...meta });
+    if (result.runId) {
+        addOutput({ type: 'info', text: `  Run ID: ${result.runId}`, eventType: 'system_message', payload, ...meta });
+    }
+    const note = result.runtimeParameterContext?.note;
+    if (result.runtimeParameterContext) {
+        emitDesktopDebug("loadmill.runtime_parameters", "execution", {
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+        }, {
+            command,
+            flowName: result.flowName,
+            flowRunId: result.runtimeParameterContext.flowRunId,
+            parameterNames: result.runtimeParameterContext.parameterNames,
+            selectedParameterNames: result.runtimeParameterContext.selectedParameterNames,
+            futureInstructions: result.runtimeParameterContext.futureInstructions,
+            noteAdded: typeof note === "string" && note.trim().length > 0,
         });
     }
-    else {
-        addOutput({ type: 'success', text: `[Loadmill] Flow "${result.flowName}" passed`, eventType: 'system_message', payload, ...meta });
-        if (result.runId) {
-            addOutput({ type: 'info', text: `  Run ID: ${result.runId}`, eventType: 'system_message', payload, ...meta });
-        }
-        const note = result.runtimeParameterContext?.note;
-        if (result.runtimeParameterContext) {
-            emitDesktopDebug("loadmill.runtime_parameters", "execution", {
-                runId: context?.runId,
-                stepId: stepContext?.stepId,
-                instructionIndex: stepContext?.instructionIndex,
-            }, {
-                command,
-                flowName: result.flowName,
-                flowRunId: result.runtimeParameterContext.flowRunId,
-                parameterNames: result.runtimeParameterContext.parameterNames,
-                selectedParameterNames: result.runtimeParameterContext.selectedParameterNames,
-                futureInstructions: result.runtimeParameterContext.futureInstructions,
-                noteAdded: typeof note === "string" && note.trim().length > 0,
-            });
-        }
-        if (typeof note === "string" && note.trim()) {
-            const selectedNames = Array.isArray(result.runtimeParameterContext.selectedParameterNames)
-                ? result.runtimeParameterContext.selectedParameterNames.filter((name) => typeof name === "string" && name.trim())
-                : [];
-            const selectedNamesText = selectedNames.length > 0
-                ? ` (${selectedNames.join(", ")})`
-                : "";
-            context?.session?.addMessage?.("system", note);
-            context?.session?.addToTranscript?.(`[Loadmill Context] ${note}`);
-            addOutput({
-                type: 'info',
-                text: `[Loadmill] Added ${result.runtimeParameterContext.selectedCount} parameter(s)${selectedNamesText} to agent context`,
-                eventType: 'system_message',
-                payload: {
-                    ...payload,
-                    loadmillFlowRunId: result.runtimeParameterContext.flowRunId,
-                    loadmillRuntimeParameterCount: result.runtimeParameterContext.selectedCount,
-                    loadmillRuntimeParameterNames: result.runtimeParameterContext.parameterNames,
-                    loadmillSelectedRuntimeParameterNames: result.runtimeParameterContext.selectedParameterNames,
-                    loadmillParameterSelectionFutureInstructions: result.runtimeParameterContext.futureInstructions,
-                },
-                ...meta
-            });
-        }
+    if (typeof note === "string" && note.trim()) {
+        const selectedNames = Array.isArray(result.runtimeParameterContext.selectedParameterNames)
+            ? result.runtimeParameterContext.selectedParameterNames.filter((name) => typeof name === "string" && name.trim())
+            : [];
+        const selectedNamesText = selectedNames.length > 0
+            ? ` (${selectedNames.join(", ")})`
+            : "";
+        context?.session?.addMessage?.("system", note);
+        context?.session?.addToTranscript?.(`[Loadmill Context] ${note}`);
+        addOutput({
+            type: 'info',
+            text: `[Loadmill] Added ${result.runtimeParameterContext.selectedCount} parameter(s)${selectedNamesText} to agent context`,
+            eventType: 'system_message',
+            payload: {
+                ...payload,
+                loadmillFlowRunId: result.runtimeParameterContext.flowRunId,
+                loadmillRuntimeParameterCount: result.runtimeParameterContext.selectedCount,
+                loadmillRuntimeParameterNames: result.runtimeParameterContext.parameterNames,
+                loadmillSelectedRuntimeParameterNames: result.runtimeParameterContext.selectedParameterNames,
+                loadmillParameterSelectionFutureInstructions: result.runtimeParameterContext.futureInstructions,
+            },
+            ...meta
+        });
     }
 }

package/build/src/device/openai.js

@@ -81,7 +81,7 @@ const closeCurrentTabTool = {
 const loadmillTool = {
     type: "function",
     name: "loadmill",
-    description: "Run an explicit Loadmill command. Use this only when the instruction explicitly mentions Loadmill and asks for a Loadmill-specific action.",
+    description: "Search Loadmill for the requested flow, run the best match, and wait for completion. Use this when the instruction explicitly mentions running or using a Loadmill flow.",
     strict: true,
     parameters: {
         type: "object",

package/build/src/integrations/loadmill/executor.js

@@ -6,7 +6,27 @@ import { interpretLoadmillCommand, selectBestFlow } from "./interpreter.js";
 import { extractLoadmillRuntimeParameters, renderLoadmillParameterContextNote, selectLoadmillParameterCandidates, } from "./parameters.js";
 import { logger } from "../../utils/logger.js";
 const POLL_INTERVAL_MS = 5000; // 5 seconds
-const TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
+const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes
+function readTimeoutMsFromEnv() {
+    const raw = process.env.LOADMILL_RUN_TIMEOUT_MS;
+    if (!raw) {
+        return DEFAULT_TIMEOUT_MS;
+    }
+    const parsed = Number.parseInt(raw, 10);
+    if (Number.isFinite(parsed) && parsed > 0) {
+        return parsed;
+    }
+    logger.debug("Ignoring invalid LOADMILL_RUN_TIMEOUT_MS value", { value: raw });
+    return DEFAULT_TIMEOUT_MS;
+}
+function formatDuration(ms) {
+    const totalSeconds = Math.ceil(ms / 1000);
+    if (totalSeconds >= 60 && totalSeconds % 60 === 0) {
+        const minutes = totalSeconds / 60;
+        return `${minutes} ${minutes === 1 ? "minute" : "minutes"}`;
+    }
+    return `${totalSeconds} ${totalSeconds === 1 ? "second" : "seconds"}`;
+}
 function buildParameterSelectionMessages({ command, flowName, futureInstructions, parameters, }) {
     const parameterList = parameters
         .map((param) => `- ${param.key}`)
@@ -127,9 +147,10 @@ function sleep(ms) {
  * @param {Function} onStatusUpdate - Callback for status updates
  * @returns {Promise<Object>} - Final run status
  */
-async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => { }) {
-    const startTime = Date.now();
-    while (Date.now() - startTime < TIMEOUT_MS) {
+async function pollForCompletion(runId, loadmillClient, options = {}) {
+    const { onStatusUpdate = () => { }, timeoutMs = readTimeoutMsFromEnv(), pollIntervalMs = POLL_INTERVAL_MS, now = Date.now, sleepFn = sleep, } = options;
+    const startTime = now();
+    while (now() - startTime < timeoutMs) {
         try {
             const status = await loadmillClient.getTestRunStatus(runId);
             onStatusUpdate(status);
@@ -158,7 +179,7 @@ async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => {
                 return { success: false, status: "stopped", result: status, error: "Test flow was stopped" };
             }
             // Still running, wait and poll again
-            await sleep(POLL_INTERVAL_MS);
+            await sleepFn(pollIntervalMs);
         }
         catch (err) {
             logger.error("Error polling Loadmill run status", { runId, error: err.message });
@@ -166,7 +187,7 @@ async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => {
         }
     }
     // Timeout
-    return { success: false, status: "timeout", error: "Test run timed out after 5 minutes" };
+    return { success: false, status: "timeout", error: `Test run timed out after ${formatDuration(timeoutMs)}` };
 }
 /**
  * Execute a Loadmill command from natural language input
@@ -176,7 +197,7 @@ async function pollForCompletion(runId, loadmillClient, onStatusUpdate = () => {
  * @returns {Promise<{success: boolean, error?: string, result?: Object}>}
  */
 export async function executeLoadmillCommand(userInput, options = {}) {
-    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, futureInstructions = [], } = options;
+    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, futureInstructions = [], pollTimeoutMs, pollIntervalMs, now, sleepFn, } = options;
     // Check for API token
     if (!loadmillClient.getApiToken()) {
         return {
@@ -214,18 +235,6 @@ export async function executeLoadmillCommand(userInput, options = {}) {
                 error: "Could not select a matching flow"
             };
         }
-        // If action is "search", just return the results
-        if (interpreted.action === "search") {
-            return {
-                success: true,
-                action: "search",
-                result: {
-                    flows,
-                    selectedFlow,
-                    confidence
-                }
-            };
-        }
         // Step 4: Run the flow
         const flowName = selectedFlow.description || selectedFlow.name || "Unknown";
         onProgress({
@@ -244,8 +253,14 @@ export async function executeLoadmillCommand(userInput, options = {}) {
         }
         // Step 5: Poll for completion
         onProgress({ step: "polling", message: `Test started (ID: ${runId}). Waiting for completion...`, runId });
-        const finalResult = await pollForCompletion(runId, loadmillClient, (status) => {
-            onProgress({ step: "polling", message: `Status: ${status.status}...`, runId });
+        const finalResult = await pollForCompletion(runId, loadmillClient, {
+            timeoutMs: pollTimeoutMs,
+            pollIntervalMs,
+            now,
+            sleepFn,
+            onStatusUpdate: (status) => {
+                onProgress({ step: "polling", message: `Status: ${status.status}...`, runId });
+            },
         });
         const runtimeParameterContext = finalResult.success
             ? await buildRuntimeParameterContext({

package/build/src/integrations/loadmill/interpreter.js

@@ -5,7 +5,7 @@ import { buildLoadmillCommandInterpretationMessages, buildLoadmillFlowSelectionM
 /**
  * Interpret a natural language Loadmill command into structured data
  * @param {string} userInput - Natural language command
- * @returns {Promise<{searchQuery: string, parameters: Object, action: 'run'|'search'}>}
+ * @returns {Promise<{searchQuery: string, parameters: Object}>}
  */
 export async function interpretLoadmillCommand(userInput, { client }) {
     const response = await client.chat.completions.create({
@@ -18,7 +18,6 @@ export async function interpretLoadmillCommand(userInput, { client }) {
     return {
         searchQuery: parsed.searchQuery || userInput,
         parameters: parsed.parameters || {},
-        action: parsed.action || "run"
     };
 }
 /**

package/build/src/prompts/execution.js

@@ -44,7 +44,7 @@ Your process:
    - Before calling \`assert\`, make sure the current screen contains the evidence needed to validate the condition. If needed, perform only minimal preparation first, such as taking a screenshot, waiting briefly, closing the keyboard, scrolling within the relevant view, or opening the relevant menu.
    - Once you call \`assert\`, validation is read-only. The assertion tool will not click, type, wait, scroll, request screenshots, or change the device state.
    - Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
-   - Use \`loadmill\` only when the instruction explicitly references the word "loadmill"
+   - Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
 3. Execute the required actions
 4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
 5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation

package/build/src/prompts/loadmill.js

@@ -7,25 +7,21 @@ export function buildLoadmillCommandInterpretationMessages(userInput) {
 Extract the following from the user's input:
 1. searchQuery: The flow name or description to search for (required). FIX any obvious typos or misspellings.
 2. parameters: Any key=value pairs mentioned (as an object)
-3. action: Either "run" (if user wants to execute) or "search" (if user just wants to find flows)
 
 Output JSON only, no markdown or explanation.
 
 Examples:
 Input: "run the checkout flow with user=test123"
-Output: {"searchQuery": "checkout flow", "parameters": {"user": "test123"}, "action": "run"}
-
-Input: "search for login test"
-Output: {"searchQuery": "login test", "parameters": {}, "action": "search"}
+Output: {"searchQuery": "checkout flow", "parameters": {"user": "test123"}}
 
 Input: "run user authentication with email=test@example.com password=secret123"
-Output: {"searchQuery": "user authentication", "parameters": {"email": "test@example.com", "password": "secret123"}, "action": "run"}
+Output: {"searchQuery": "user authentication", "parameters": {"email": "test@example.com", "password": "secret123"}}
 
 Input: "execute payment flow"
-Output: {"searchQuery": "payment flow", "parameters": {}, "action": "run"}
+Output: {"searchQuery": "payment flow", "parameters": {}}
 
 Input: "create a transction with amount=200"
-Output: {"searchQuery": "transaction", "parameters": {"amount": "200"}, "action": "run"}`
+Output: {"searchQuery": "transaction", "parameters": {"amount": "200"}}`
         },
         {
             role: "user",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.14.0",
+  "version": "2.15.1",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -24,6 +24,9 @@
     "build": "tsc",
     "clean": "rm -rf build",
     "typecheck": "tsc --noEmit",
+    "typecheck:desktop": "npm --prefix desktop-app run typecheck",
+    "typecheck:all": "npm run typecheck && npm run typecheck:desktop",
+    "prepare": "node scripts/install-git-hooks.mjs",
     "prepublishOnly": "npm run build"
   },
   "repository": {