@loadmill/droid-cua 2.15.1 → 2.17.0

package/README.md

@@ -46,9 +46,9 @@ Under the hood, `droid-cua` uses an AI agent to explore your app, execute action
 
 Open the desktop app and choose the platform you want to test.
 
-**3. Add your credentials**
+**3. Choose your AI provider**
 
-Set your OpenAI API key in the app Settings screen, or log in with your Loadmill account.
+In Settings, choose the CUA model you want to run. `Loadmill - Smart` uses your Loadmill sign-in and usage tracking. The OpenAI models use your own OpenAI API key.
 
 **4. Connect a target device**
 
@@ -75,7 +75,7 @@ Desktop app support:
 ## Requirements
 
 **All platforms:**
-- OpenAI API key
+- Loadmill sign-in for `Loadmill - Smart`, or an OpenAI API key for the OpenAI models
 
 **Android:**
 - Android Debug Bridge (ADB)

package/build/index.js

@@ -13,6 +13,7 @@ import { createOpenAiExecutor } from "./src/device/openai.js";
 import { logger } from "./src/utils/logger.js";
 import { selectDevice } from "./src/cli/device-selector.js";
 import { resolveExecutionAppContext } from "./src/core/app-context.js";
+import { resolveExecutionTestData } from "./src/core/test-data.js";
 import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
 import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
@@ -20,6 +21,7 @@ import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
 import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
 import { listHeadlessInstructionFiles, resolveHeadlessInstructionsTarget } from "./src/cli/headless-instruction-target.js";
 import { runHeadlessInstructionRuns } from "./src/cli/headless-runner.js";
+import { planScenarioRuns } from "./src/cli/scenario-planner.js";
 import { resolveLambdaTestCredentials, uploadLambdaTestCliApp } from "./src/cli/lambdatest-app-upload.js";
 import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
 import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
@@ -29,6 +31,7 @@ const targetConfig = resolveHeadlessTarget(args);
 let avdName = targetConfig.avdName;
 let platform = targetConfig.platform || null; // 'ios' or 'android'
 const recordScreenshots = args["record"] || false;
+const startupUrl = typeof args.url === "string" && args.url.trim().length > 0 ? args.url.trim() : null;
 const instructionsFile = args.instructions || args.i || null;
 const appContextPath = typeof args.context === "string" ? args.context : null;
 const debugMode = args["debug"] || false;
@@ -66,6 +69,41 @@ function wait(ms) {
 function isLambdaTestAppAutomationUrl(value) {
     return typeof value === "string" && value.startsWith("https://appautomation.lambdatest.com/test?");
 }
+async function loadScenarioCandidateRuns(runs) {
+    return await Promise.all(runs.map(async (run) => ({
+        ...run,
+        instructions: run.instructions ?? await loadHeadlessInstructions(run.testPath),
+    })));
+}
+function printScenarioCasePreview(cases) {
+    console.log(`\nScenario preview: ${cases.length} case${cases.length === 1 ? "" : "s"}\n`);
+    cases.forEach((entry, index) => {
+        console.log(`${index + 1}. ${entry.caseName}`);
+    });
+    console.log("");
+}
+function buildScenarioInstructionTarget(originalTarget, scenarioPrompt) {
+    return {
+        type: "scenario",
+        path: originalTarget?.path || null,
+        sourceType: originalTarget?.type || null,
+        scenarioPrompt,
+    };
+}
+function buildScenarioRuns({ candidateRuns, cases }) {
+    const runByTestName = new Map(candidateRuns.map((run) => [run.testName, run]));
+    return cases.map((entry) => {
+        const sourceRun = runByTestName.get(entry.testName);
+        return {
+            ...sourceRun,
+            testName: entry.caseName,
+            sourceTestName: entry.testName,
+            scenarioCaseName: entry.caseName,
+            testDataContext: entry.testDataContext,
+            instructions: sourceRun.instructions,
+        };
+    });
+}
 async function resolveLambdaTestDashboardUrl(credentials, sessionId, { attempts = 1, delayMs = 1000 } = {}) {
     const adapter = getCloudProviderAdapter("lambdatest");
     for (let attempt = 1; attempt <= attempts; attempt += 1) {
@@ -106,10 +144,17 @@ async function main() {
     if (isLambdaTestTarget && !isHeadlessInstructionsRun) {
         throw new Error("--device-source lambdatest is only supported with --instructions headless execution.");
     }
+    if (!isHeadlessInstructionsRun && args.scenario != null) {
+        throw new Error("--scenario can only be used with --instructions headless execution.");
+    }
+    if (!isHeadlessInstructionsRun && args["test-data"] != null) {
+        throw new Error("--test-data can only be used with --instructions headless execution.");
+    }
     let preloadedHeadlessConfig = null;
     let preloadedHeadlessTarget = null;
     let headlessInstructionRuns = [];
     let preloadedAppContext = null;
+    let preloadedTestData = null;
     if (isHeadlessInstructionsRun) {
         preloadedHeadlessConfig = await resolveHeadlessExecutionConfig(args);
         preloadedHeadlessTarget = await resolveHeadlessInstructionsTarget(instructionsFile);
@@ -138,6 +183,14 @@ async function main() {
         if (preloadedAppContext.status === "failed") {
             throw new Error(`Could not read app context file ${preloadedAppContext.contextPath}: ${preloadedAppContext.message}`);
         }
+        if (preloadedHeadlessConfig.scenarioPrompt) {
+            preloadedTestData = await resolveExecutionTestData({
+                testDataPath: preloadedHeadlessConfig.testDataPath,
+            });
+            if (preloadedTestData.status === "failed") {
+                throw new Error(`Could not read test data file ${preloadedTestData.testDataPath}: ${preloadedTestData.message}`);
+            }
+        }
     }
     // If no device specified, show interactive selection menu
     if (!isBrowserTarget && !isLambdaTestTarget && !avdName && !platform) {
@@ -149,6 +202,7 @@ async function main() {
     let deviceInfo;
     if (isBrowserTarget) {
         targetId = await connectToBrowser({
+            url: startupUrl,
             headless: true,
             browserFamily: targetConfig.browserFamily,
             sessionMode: targetConfig.sessionMode
@@ -219,6 +273,7 @@ async function main() {
             const appContext = preloadedAppContext;
             const appContextText = appContext.text;
             const runtimeContext = createCliRuntimeContext({
+                llmProviderMode: headlessConfig.llmProviderMode,
                 cuaModel: headlessConfig.cuaModel,
                 promptCustomizations: headlessConfig.promptCustomizations,
                 contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
@@ -231,6 +286,28 @@ async function main() {
                     message: appContext.message,
                 },
             });
+            console.log(`Using AI provider: ${runtimeContext.config.llmProviderMode}`);
+            console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
+            if (headlessConfig.scenarioPrompt) {
+                const candidateRuns = await loadScenarioCandidateRuns(headlessInstructionRuns);
+                const scenarioCases = await planScenarioRuns({
+                    client: runtimeContext.openAiClient,
+                    runtimeConfig: runtimeContext.config,
+                    scenarioPrompt: headlessConfig.scenarioPrompt,
+                    tests: candidateRuns.map((run) => ({
+                        testName: run.testName,
+                        instructions: run.instructions,
+                    })),
+                    appContextText,
+                    testDataText: preloadedTestData?.text || "",
+                });
+                printScenarioCasePreview(scenarioCases);
+                headlessInstructionRuns = buildScenarioRuns({
+                    candidateRuns,
+                    cases: scenarioCases,
+                });
+                preloadedHeadlessTarget = buildScenarioInstructionTarget(preloadedHeadlessTarget, headlessConfig.scenarioPrompt);
+            }
             const result = await runHeadlessInstructionRuns({
                 runs: headlessInstructionRuns,
                 instructionsTarget: preloadedHeadlessTarget,

package/build/src/cli/headless-execution-config.js

@@ -2,7 +2,8 @@ import path from "path";
 import { readFile } from "fs/promises";
 import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
 import { AIM_ASSIST_MODES, DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
-import { CUA_MODELS, isValidCuaModel, resolveCuaModel } from "../core/cua-models.js";
+import { CUA_MODELS, DEFAULT_OPENAI_CUA_MODEL, isOpenAiCuaModel, isValidCuaModel, resolveCuaModel } from "../core/cua-models.js";
+import { LLM_PROVIDER_MODES, isValidLlmProviderMode, resolveLlmProviderMode } from "../core/runtime-config.js";
 function createEmptyPromptCustomizations() {
     return {
         basePromptInstructions: "",
@@ -30,8 +31,25 @@ function validateAimAssistMode(value, label) {
     }
     return value;
 }
+function validateLlmProviderMode(value, label) {
+    if (typeof value !== "string" || !isValidLlmProviderMode(value)) {
+        throw new Error(`${label} must be one of: ${LLM_PROVIDER_MODES.join(", ")}.`);
+    }
+    return value;
+}
 function readEnvDefaultCuaModel() {
-    return resolveCuaModel(process.env.OPENAI_CUA_MODEL);
+    const envValue = process.env.OPENAI_CUA_MODEL;
+    // OPENAI_CUA_MODEL only selects OpenAI models. If it is set but not a valid
+    // OpenAI model, stay on the OpenAI default rather than crossing over to the
+    // global (Loadmill) default.
+    if (typeof envValue === "string" && envValue.trim()) {
+        const trimmed = envValue.trim();
+        return isOpenAiCuaModel(trimmed) ? trimmed : DEFAULT_OPENAI_CUA_MODEL;
+    }
+    return resolveCuaModel(envValue);
+}
+function readEnvDefaultLlmProviderMode() {
+    return resolveLlmProviderMode(process.env.DROID_CUA_LLM_PROVIDER);
 }
 function parseContextOptimizationThreshold(rawValue, label) {
     const numericValue = (() => {
@@ -107,6 +125,7 @@ async function loadConfigFromFile(configPath) {
     const configDir = path.dirname(absoluteConfigPath);
     const normalized = {
         configPath: absoluteConfigPath,
+        llmProviderMode: undefined,
         cuaModel: undefined,
         aimAssistMode: undefined,
         promptCustomizations: createEmptyPromptCustomizations(),
@@ -115,6 +134,9 @@ async function loadConfigFromFile(configPath) {
         contextOptimizationEnabled: undefined,
         contextOptimizationThreshold: undefined,
     };
+    if ("llmProviderMode" in rawConfig) {
+        normalized.llmProviderMode = validateLlmProviderMode(rawConfig.llmProviderMode, "config.llmProviderMode");
+    }
     if ("cuaModel" in rawConfig) {
         normalized.cuaModel = validateCuaModel(rawConfig.cuaModel, "config.cuaModel");
     }
@@ -150,6 +172,10 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
     const cwd = typeof options.cwd === "string" ? options.cwd : process.cwd();
     const configPath = typeof args.config === "string" ? args.config : null;
     const explicitContextPath = typeof args.context === "string" ? path.resolve(cwd, args.context) : null;
+    const scenarioPrompt = typeof args.scenario === "string" ? args.scenario.trim() : null;
+    const hasScenarioFlag = args.scenario != null;
+    const testDataPath = typeof args["test-data"] === "string" ? path.resolve(cwd, args["test-data"]) : null;
+    const hasTestDataFlag = args["test-data"] != null;
     const noContext = args["no-context"] === true || args.context === false;
     const contextOptimizationEnabledByFlag = args["context-optimization"] === true;
     const noContextOptimization = args["no-context-optimization"] === true || args["context-optimization"] === false;
@@ -161,6 +187,15 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
     if (explicitContextPath && noContext) {
         throw new Error("--context and --no-context cannot be used together.");
     }
+    if (hasScenarioFlag && !scenarioPrompt) {
+        throw new Error("--scenario must be a non-empty string.");
+    }
+    if (hasTestDataFlag && !testDataPath) {
+        throw new Error("--test-data must be a non-empty path.");
+    }
+    if (testDataPath && !scenarioPrompt) {
+        throw new Error("--test-data can only be used with --scenario.");
+    }
     if (contextOptimizationEnabledByFlag) {
         throw new Error("--context-optimization is not supported; Context Optimization is enabled by default.");
     }
@@ -174,14 +209,20 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
     };
     const resolved = {
         configPath: fileConfig?.configPath || null,
+        llmProviderMode: fileConfig?.llmProviderMode || readEnvDefaultLlmProviderMode(),
         cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
         aimAssistMode: fileConfig?.aimAssistMode || DEFAULT_AIM_ASSIST_MODE,
         promptCustomizations,
         appContextEnabled: fileConfig?.appContextEnabled ?? true,
         appContextPath: fileConfig?.appContextPath || null,
+        scenarioPrompt,
+        testDataPath,
         contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
         contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
     };
+    if (typeof args["llm-provider"] === "string") {
+        resolved.llmProviderMode = validateLlmProviderMode(args["llm-provider"], "--llm-provider");
+    }
     if (typeof args["cua-model"] === "string") {
         resolved.cuaModel = validateCuaModel(args["cua-model"], "--cua-model");
     }

package/build/src/cli/headless-runner.js

@@ -100,11 +100,12 @@ function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionP
     }
     return engineOptions;
 }
-async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
+async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
     let executionMode = null;
     try {
         await headlessDebug.startExecutionSession(runId, {
             testName,
+            ...(testDataContext ? { testDataUsed: true } : {}),
             platform: deviceInfo.platform,
             deviceName: deviceInfo.device_name,
             aimAssistMode: headlessConfig.aimAssistMode
@@ -119,7 +120,10 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         emitCliAppContextDebug(runId, appContext, appContextText);
         session.updateResponseId(undefined);
         session.clearMessages();
-        const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { aimAssistMode: headlessConfig.aimAssistMode });
+        const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, {
+            aimAssistMode: headlessConfig.aimAssistMode,
+            testDataContext,
+        });
         session.setSystemPrompt(executionPrompt);
         const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
             runId,
@@ -188,9 +192,12 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
         throw error;
     }
 }
-export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
-    const isBatchRun = instructionsTarget?.type === "directory";
-    if (isBatchRun) {
+export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget, runHeadlessTestImpl = runHeadlessTest }) {
+    const isBatchRun = instructionsTarget?.type === "directory" || instructionsTarget?.type === "scenario";
+    if (instructionsTarget?.type === "scenario") {
+        console.log(`\nRunning ${runs.length} scenario case${runs.length === 1 ? "" : "s"}\n`);
+    }
+    else if (isBatchRun) {
         console.log(`\nRunning ${runs.length} tests from: ${instructionsTarget.path}\n`);
     }
     let failures = 0;
@@ -205,7 +212,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
             if (recordScreenshots && testScreenshotDir) {
                 await mkdir(testScreenshotDir, { recursive: true });
             }
-            const result = await runHeadlessTest({
+            const result = await runHeadlessTestImpl({
                 runId,
                 testPath: run.testPath,
                 testName: run.testName,
@@ -214,6 +221,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
                 deviceInfo,
                 headlessConfig,
                 appContext,
+                testDataContext: run.testDataContext || "",
                 runtimeContext,
                 headlessDebug,
                 recordScreenshots,
@@ -234,7 +242,11 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
             console.error(`\nTest failed: ${run.testName}: ${message}`);
         }
     }
-    if (isBatchRun) {
+    if (instructionsTarget?.type === "scenario") {
+        const passed = runs.length - failures;
+        console.log(`\nScenario run complete: ${passed}/${runs.length} passed`);
+    }
+    else if (isBatchRun) {
         const passed = runs.length - failures;
         console.log(`\nFolder run complete: ${passed}/${runs.length} passed`);
     }

package/build/src/cli/ink-shell.js

@@ -35,6 +35,7 @@ export async function startInkShell(session, executionEngine, options = {}) {
                 engine: executionEngine,
                 appContextPath: options.appContextPath ?? null,
                 runtimeContext: options.runtimeContext ?? null,
+                runtimeConfig: options.runtimeContext?.config ?? null,
                 openAiClient: options.runtimeContext?.openAiClient ?? null,
                 loadmillClient: options.runtimeContext?.loadmillClient ?? null,
                 runtimeCredentials: options.runtimeContext?.credentials ?? null

package/build/src/cli/scenario-planner.js

@@ -0,0 +1 @@
+export { planScenarioRuns, validateScenarioCases, } from "../core/scenario-planner.js";

package/build/src/commands/help.js

@@ -26,9 +26,12 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --os-version <version> LambdaTest OS version' });
     addOutput({ type: 'info', text: '  --app <path>           LambdaTest app package path (.apk or .ipa)' });
     addOutput({ type: 'info', text: '  --instructions <file>  Run test file in headless mode' });
+    addOutput({ type: 'info', text: '  --scenario <prompt>    Generate and run scenario cases for headless instructions' });
+    addOutput({ type: 'info', text: '  --test-data <file>     Optional test-data.md file for --scenario' });
     addOutput({ type: 'info', text: '  --session-mode <mode>  Browser session mode for web runs: persistent or fresh' });
     addOutput({ type: 'info', text: '  --config <file>        Headless execution JSON config for prompt parity' });
-    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4 or gpt-5.4-mini' });
+    addOutput({ type: 'info', text: '  --llm-provider <mode>  Headless AI provider: loadmill or openai' });
+    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4, loadmill-smart, or gpt-5.4-mini' });
     addOutput({ type: 'info', text: '  --aim-assist <mode>    Headless Aim Assist mode: off, smart, or always' });
     addOutput({ type: 'info', text: '  --context <file>       Optional app context file injected into execution runs' });
     addOutput({ type: 'info', text: '  --no-context           Disable app context for headless execution' });
@@ -79,6 +82,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  droid-cua --avd "iPhone 16"             (iOS Simulator, auto-detected)' });
     addOutput({ type: 'info', text: '  droid-cua --platform ios --avd MySim    (Force iOS platform)' });
     addOutput({ type: 'info', text: '  droid-cua --instructions tests/login.dcua --context app/context.md' });
+    addOutput({ type: 'info', text: '  droid-cua --instructions tests --test-data tests/test-data.md --scenario "Run every checkout test with each buyer account"' });
     addOutput({ type: 'info', text: '  droid-cua --target web --instructions tests/search.dcua' });
     addOutput({ type: 'info', text: '  droid-cua --device-source lambdatest --platform android --device-name "Galaxy S24" --os-version 14 --app app.apk --instructions tests/login.dcua' });
     addOutput({ type: 'info', text: '  /create login-test                      (design a new test)' });

package/build/src/commands/loadmill.js

@@ -2,7 +2,7 @@
  * Loadmill command handler
  */
 import { createLoadmillApiClient, executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
-import { createOpenAiClient, resolveRuntimeCredentialsFromEnv } from "../core/runtime-config.js";
+import { createResolvedLlmClient, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "../core/runtime-config.js";
 /**
  * Handle /loadmill command
  * @param {string} args - Command arguments (natural language command)
@@ -48,33 +48,60 @@ export async function handleLoadmill(args, session, context) {
     if (context.setAgentWorking) {
         context.setAgentWorking(true, 'Running Loadmill flow...');
     }
-    const runtimeCredentials = context.runtimeCredentials || resolveRuntimeCredentialsFromEnv();
-    const loadmillClient = context.loadmillClient || createLoadmillApiClient({
-        token: runtimeCredentials.loadmillApiToken,
-        baseUrl: runtimeCredentials.loadmillBaseUrl,
-    });
-    const openAiClient = context.openAiClient || createOpenAiClient(runtimeCredentials.openAiApiKey);
-    // Execute the command
-    const result = await executeLoadmillCommand(command, {
-        loadmillClient,
-        openAiClient,
-        onProgress: ({ message }) => {
-            addOutput({ type: 'info', text: `[Loadmill] ${message}` });
+    try {
+        const runtimeCredentials = context.runtimeCredentials || resolveRuntimeCredentialsFromEnv();
+        const runtimeConfig = context.runtimeConfig || resolveRuntimeConfigFromEnv();
+        const loadmillClient = context.loadmillClient || createLoadmillApiClient({
+            token: runtimeCredentials.loadmillApiToken,
+            baseUrl: runtimeCredentials.loadmillBaseUrl,
+        });
+        const openAiClient = context.openAiClient || createResolvedLlmClient({
+            llmProviderMode: runtimeConfig.llmProviderMode,
+            credentials: runtimeCredentials,
+            clientName: "cli",
+        });
+        // Execute the command
+        const result = await executeLoadmillCommand(command, {
+            loadmillClient,
+            openAiClient,
+            onProgress: ({ message }) => {
+                addOutput({ type: 'info', text: `[Loadmill] ${message}` });
+            }
+        });
+        // Display results
+        if (result.success) {
+            if (result.action === "search") {
+                addOutput({ type: 'success', text: `Found ${result.result.flows.length} flow(s):` });
+                result.result.flows.forEach((flow, i) => {
+                    const name = flow.description || flow.name || "Unknown";
+                    const suite = flow.testSuiteDescription ? ` (Suite: ${flow.testSuiteDescription})` : '';
+                    addOutput({ type: 'info', text: `  ${i + 1}. ${name}${suite}` });
+                    addOutput({ type: 'info', text: `     ID: ${flow.id}` });
+                });
+                if (result.result.selectedFlow) {
+                    const selectedName = result.result.selectedFlow.description || result.result.selectedFlow.name || "Unknown";
+                    addOutput({ type: 'info', text: '' });
+                    addOutput({ type: 'info', text: `Best match: "${selectedName}" (confidence: ${(result.result.confidence * 100).toFixed(0)}%)` });
+                }
+            }
+            else {
+                addOutput({ type: 'success', text: `Flow "${result.flowName}" passed` });
+                if (result.runId) {
+                    addOutput({ type: 'info', text: `Run ID: ${result.runId}` });
+                }
+            }
         }
-    });
-    // Clear agent working status
-    if (context.setAgentWorking) {
-        context.setAgentWorking(false);
-    }
-    // Display results
-    if (result.success) {
-        addOutput({ type: 'success', text: `Flow "${result.flowName}" passed` });
-        if (result.runId) {
-            addOutput({ type: 'info', text: `Run ID: ${result.runId}` });
+        else {
+            addOutput({ type: 'error', text: `Loadmill failed: ${result.error}` });
         }
     }
-    else {
-        addOutput({ type: 'error', text: `Loadmill failed: ${result.error}` });
+    catch (error) {
+        addOutput({ type: 'error', text: error instanceof Error ? error.message : 'Loadmill command failed.' });
+    }
+    finally {
+        if (context.setAgentWorking) {
+            context.setAgentWorking(false);
+        }
     }
     return true; // Continue loop
 }

package/build/src/core/cua-backends.js

@@ -0,0 +1,11 @@
+export const CUA_BACKENDS = ["openai", "fara"];
+export const DEFAULT_CUA_BACKEND = "openai";
+export const DEFAULT_FARA_BASE_URL = "http://localhost:5000/v1";
+export const DEFAULT_FARA_MODEL = "Fara-7B";
+const VALID_CUA_BACKENDS = new Set(CUA_BACKENDS);
+export function isValidCuaBackend(value) {
+    return VALID_CUA_BACKENDS.has(value);
+}
+export function resolveCuaBackend(value) {
+    return isValidCuaBackend(value) ? value : DEFAULT_CUA_BACKEND;
+}

package/build/src/core/cua-models.js

@@ -1,9 +1,54 @@
-export const CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
-export const DEFAULT_CUA_MODEL = "gpt-5.4";
+export const DEFAULT_OPENAI_CUA_MODEL = "gpt-5.4";
+export const DEFAULT_LOADMILL_CUA_MODEL = "loadmill-smart";
+export const DEFAULT_CUA_MODEL = DEFAULT_LOADMILL_CUA_MODEL;
+export const OPENAI_CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
+export const LOADMILL_CUA_MODELS = [DEFAULT_LOADMILL_CUA_MODEL];
+export const CUA_MODELS = ["gpt-5.4", "loadmill-smart", "gpt-5.4-mini"];
+export const CUA_MODEL_OPTIONS = Object.freeze([
+    Object.freeze({
+        id: "loadmill-smart",
+        providerMode: "loadmill",
+        providerLabel: "Loadmill",
+        name: "Loadmill Smart",
+        label: "Loadmill Smart (recommended)",
+    }),
+    Object.freeze({
+        id: "gpt-5.4",
+        providerMode: "openai",
+        providerLabel: "OpenAI",
+        name: "GPT-5.4",
+        label: "OpenAI GPT-5.4",
+    }),
+    Object.freeze({
+        id: "gpt-5.4-mini",
+        providerMode: "openai",
+        providerLabel: "OpenAI",
+        name: "GPT-5.4 mini",
+        label: "OpenAI GPT-5.4 mini (faster)",
+    }),
+]);
 const VALID_CUA_MODELS = new Set(CUA_MODELS);
+const OPENAI_CUA_MODEL_SET = new Set(OPENAI_CUA_MODELS);
+const LOADMILL_CUA_MODEL_SET = new Set(LOADMILL_CUA_MODELS);
+const CUA_MODEL_PROVIDER_MODE_MAP = new Map(CUA_MODEL_OPTIONS.map((option) => [option.id, option.providerMode]));
 export function isValidCuaModel(value) {
     return VALID_CUA_MODELS.has(value);
 }
+export function isOpenAiCuaModel(value) {
+    return OPENAI_CUA_MODEL_SET.has(value);
+}
+export function isLoadmillCuaModel(value) {
+    return LOADMILL_CUA_MODEL_SET.has(value);
+}
 export function resolveCuaModel(value) {
     return isValidCuaModel(value) ? value : DEFAULT_CUA_MODEL;
 }
+export function resolveCuaModelProviderMode(value) {
+    const normalized = resolveCuaModel(value);
+    return CUA_MODEL_PROVIDER_MODE_MAP.get(normalized) || "openai";
+}
+export function resolveDefaultCuaModelForProvider(providerMode) {
+    return providerMode === "loadmill"
+        ? DEFAULT_LOADMILL_CUA_MODEL
+        : DEFAULT_OPENAI_CUA_MODEL;
+}

package/build/src/core/prompts.js

@@ -1,3 +1,3 @@
 export { buildBaseSystemPrompt } from "../prompts/base.js";
 export { buildDesignModePrompt, buildDesignRecoveryPrompt, buildDesignScriptGenerationPrompt, } from "../prompts/design.js";
-export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";
+export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, buildTestDataContextSection, } from "../prompts/execution.js";