@loadmill/droid-cua 2.8.0 → 2.9.0

package/build/src/core/execution-engine.js

@@ -9,7 +9,6 @@ import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { printCliOutput } from "../utils/console-output.js";
 import { annotateScreenshotArtifact, createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
 import { CAPTURE_SOURCES } from "./capture-sources.js";
-const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
 function extractActionableOutputItems(items) {
     const entries = [];
     for (const item of items) {
@@ -189,13 +188,14 @@ export class ExecutionEngine {
         this.screenshotDir = options.screenshotDir || null;
         this.screenshotRecorder = options.screenshotRecorder || null;
         this.strictMode = options.strictMode === true;
-        this.stepDelayMs = getConfiguredStepDelayMs();
+        this.stepDelayMs = options.stepDelayMs ?? getConfiguredStepDelayMs();
         this.reportedScreenshotWriteError = false;
         this.reportedScreenshotAnnotateError = false;
         this.lastRecordedScreenshotPaths = [];
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
+        this.reviseTestScript = options.reviseTestScript || null;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
     }
     async saveScreenshotArtifact(screenshotBase64, metadata = {}, options = {}) {
@@ -579,9 +579,6 @@ export class ExecutionEngine {
                     note: runtimeNote
                 });
             }
-            const selectedCuaModel = VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL)
-                ? process.env.OPENAI_CUA_MODEL
-                : "gpt-5.4";
             const input = [{
                     type: "computer_call_output",
                     call_id,
@@ -589,9 +586,6 @@ export class ExecutionEngine {
                         type: "computer_screenshot",
                         image_url: `data:image/png;base64,${screenshotBase64}`,
                     },
-                    ...(selectedCuaModel === "computer-use-preview"
-                        ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
-                        : {}),
                     ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
                 }];
             if (runtimeNote) {

package/build/src/core/runtime-config.js

@@ -0,0 +1,96 @@
+import OpenAI from "openai";
+import { readContextOptimizationEnv, normalizeContextOptimizationEnabled, normalizeContextOptimizationThreshold, } from "./context-optimization.js";
+import { resolveCuaModel } from "./cua-models.js";
+import { normalizeStepDelayMs } from "../utils/step-delay.js";
+export function createEmptyPromptCustomizations() {
+    return {
+        basePromptInstructions: "",
+        designModeInstructions: "",
+        executionModeInstructions: "",
+    };
+}
+export function resolvePromptCustomizations(rawValue = null) {
+    const resolved = createEmptyPromptCustomizations();
+    if (!rawValue || typeof rawValue !== "object" || Array.isArray(rawValue)) {
+        return resolved;
+    }
+    for (const key of Object.keys(resolved)) {
+        if (typeof rawValue[key] === "string") {
+            resolved[key] = rawValue[key];
+        }
+    }
+    return resolved;
+}
+function trimNullableString(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const trimmed = value.trim();
+    return trimmed || null;
+}
+export function resolveRuntimeCredentials({ openAiApiKey = null, loadmillApiToken = null, loadmillBaseUrl = null, } = {}) {
+    return {
+        openAiApiKey: trimNullableString(openAiApiKey),
+        loadmillApiToken: trimNullableString(loadmillApiToken),
+        loadmillBaseUrl: trimNullableString(loadmillBaseUrl),
+    };
+}
+export function resolveRuntimeConfig({ cuaModel = null, strictMode = false, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
+    return {
+        cuaModel: resolveCuaModel(cuaModel),
+        strictMode: strictMode === true,
+        stepDelayMs: normalizeStepDelayMs(Number(stepDelayMs)),
+        contextOptimizationEnabled: normalizeContextOptimizationEnabled(contextOptimizationEnabled),
+        contextOptimizationThreshold: normalizeContextOptimizationThreshold(contextOptimizationThreshold),
+        promptCustomizations: resolvePromptCustomizations(promptCustomizations),
+        appContext: appContext && typeof appContext === "object"
+            ? {
+                enabled: appContext.enabled !== false,
+                path: trimNullableString(appContext.path ?? appContext.contextPath),
+                status: typeof appContext.status === "string" ? appContext.status : null,
+                text: typeof appContext.text === "string" ? appContext.text : "",
+                message: trimNullableString(appContext.message),
+            }
+            : {
+                enabled: true,
+                path: null,
+                status: null,
+                text: "",
+                message: null,
+            },
+    };
+}
+export function resolveRuntimeConfigFromEnv(env = process.env, overrides = {}) {
+    const contextOptimization = readContextOptimizationEnv(env);
+    return resolveRuntimeConfig({
+        cuaModel: env.OPENAI_CUA_MODEL,
+        stepDelayMs: env.DROID_CUA_STEP_DELAY_MS,
+        contextOptimizationEnabled: contextOptimization.enabled,
+        contextOptimizationThreshold: contextOptimization.threshold,
+        ...overrides,
+    });
+}
+export function resolveRuntimeCredentialsFromEnv(env = process.env, overrides = {}) {
+    return resolveRuntimeCredentials({
+        openAiApiKey: env.OPENAI_API_KEY,
+        loadmillApiToken: env.LOADMILL_API_TOKEN,
+        loadmillBaseUrl: env.LOADMILL_BASE_URL,
+        ...overrides,
+    });
+}
+export function createOpenAiClient(apiKey) {
+    return new OpenAI({
+        apiKey: trimNullableString(apiKey) || undefined,
+    });
+}
+export function createRunContext({ config, credentials }) {
+    const resolvedConfig = resolveRuntimeConfig(config);
+    const resolvedCredentials = resolveRuntimeCredentials(credentials);
+    return {
+        config: resolvedConfig,
+        credentials: resolvedCredentials,
+        clients: {
+            openai: createOpenAiClient(resolvedCredentials.openAiApiKey),
+        },
+    };
+}

package/build/src/device/loadmill.js

@@ -3,8 +3,8 @@
  */
 import { executeLoadmillCommand } from "../integrations/loadmill/index.js";
 import { printCliOutput } from "../utils/console-output.js";
-function getLoadmillSiteBaseUrl() {
-    const rawBaseUrl = process.env.LOADMILL_BASE_URL || "https://app.loadmill.com/api";
+function getLoadmillSiteBaseUrl(loadmillClient = null) {
+    const rawBaseUrl = loadmillClient?.getBaseUrl?.() || "https://app.loadmill.com/api";
     return rawBaseUrl.replace(/\/api\/?$/, "");
 }
 /**
@@ -45,6 +45,8 @@ export function extractLoadmillCommand(userInput) {
  */
 export async function executeLoadmillInstruction(command, isHeadlessMode, context, stepContext = null) {
     const addOutput = context?.addOutput || printCliOutput;
+    const loadmillClient = context?.loadmillClient || null;
+    const openAiClient = context?.openAiClient || null;
     const meta = {
         runId: context?.runId,
         stepId: stepContext?.stepId,
@@ -54,14 +56,16 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
         type: 'info',
         text: `[Loadmill] Executing: ${command}`,
         eventType: 'system_message',
-        payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() },
+        payload: { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) },
         ...meta
     });
     const result = await executeLoadmillCommand(command, {
+        loadmillClient,
+        openAiClient,
         onProgress: ({ message, runId }) => {
             const payload = runId
-                ? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
-                : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
+                ? { loadmillSuiteRunId: runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
+                : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
             addOutput({ type: 'info', text: `[Loadmill] ${message}`, eventType: 'system_message', payload, ...meta });
         }
     });
@@ -84,14 +88,15 @@ export async function executeLoadmillInstruction(command, isHeadlessMode, contex
  */
 export async function handleLoadmillFailure(command, error, isHeadlessMode, context, stepContext = null, suiteRunId = null) {
     const addOutput = context?.addOutput || printCliOutput;
+    const loadmillClient = context?.loadmillClient || null;
     const meta = {
         runId: context?.runId,
         stepId: stepContext?.stepId,
         instructionIndex: stepContext?.instructionIndex
     };
     const payload = suiteRunId
-        ? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
-        : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
+        ? { loadmillSuiteRunId: suiteRunId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
+        : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
     addOutput({ type: 'error', text: '[Loadmill] FAILED', eventType: 'error', payload, ...meta });
     addOutput({ type: 'error', text: `Command: ${command}`, eventType: 'error', payload, ...meta });
     addOutput({ type: 'error', text: `Error: ${error}`, eventType: 'error', payload, ...meta });
@@ -148,14 +153,15 @@ export async function handleLoadmillFailure(command, error, isHeadlessMode, cont
  */
 export function handleLoadmillSuccess(command, result, context, stepContext = null) {
     const addOutput = context?.addOutput || printCliOutput;
+    const loadmillClient = context?.loadmillClient || null;
     const meta = {
         runId: context?.runId,
         stepId: stepContext?.stepId,
         instructionIndex: stepContext?.instructionIndex
     };
     const payload = result.runId
-        ? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() }
-        : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl() };
+        ? { loadmillSuiteRunId: result.runId, loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) }
+        : { loadmillCommand: command, loadmillBaseUrl: getLoadmillSiteBaseUrl(loadmillClient) };
     if (result.action === "search") {
         addOutput({ type: 'success', text: `[Loadmill] Found ${result.result.flows.length} flow(s)`, eventType: 'system_message', payload, ...meta });
         result.result.flows.forEach((flow, i) => {

package/build/src/device/openai.js

@@ -1,15 +1,9 @@
-import OpenAI from "openai";
-import { readContextOptimizationEnv } from "../core/context-optimization.js";
 import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
-import { buildAppContextCompactionInput } from "../prompts/execution.js";
+import { createOpenAiClient, resolveRuntimeConfig } from "../core/runtime-config.js";
+import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD } from "../core/context-optimization.js";
 import { logger } from "../utils/logger.js";
 import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
-let openai = null;
 const cuaDebugTracer = new CuaDebugTracer(logger);
-const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
-function getSelectedCuaModel() {
-    return VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL) ? process.env.OPENAI_CUA_MODEL : "gpt-5.4";
-}
 function normalizeMessageContent(content) {
     if (Array.isArray(content)) {
         return content.map((item) => {
@@ -90,37 +84,14 @@ export function buildCuaInput({ messages, screenshotBase64, previousResponseId,
     }
     return input;
 }
-function buildContextManagement(cuaModel) {
-    if (cuaModel === "computer-use-preview") {
-        return undefined;
-    }
-    const { enabled, threshold } = readContextOptimizationEnv();
-    if (!enabled) {
+function buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold) {
+    if (!contextOptimizationEnabled) {
         return undefined;
     }
-    return [{ type: "compaction", compact_threshold: threshold }];
+    return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
 }
-export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
-    const contextManagement = buildContextManagement(cuaModel);
-    if (cuaModel === "computer-use-preview") {
-        return {
-            model: cuaModel,
-            previous_response_id: previousResponseId || undefined,
-            input,
-            store: true,
-            truncation: "auto",
-            tools: [
-                {
-                    type: "computer_use_preview",
-                    display_width: deviceInfo.scaled_width,
-                    display_height: deviceInfo.scaled_height,
-                    environment: "browser",
-                },
-                previewClickTool,
-            ],
-            reasoning: { summary: "concise" },
-        };
-    }
+export function buildCuaRequestParams({ cuaModel, previousResponseId, input, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, }) {
+    const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
     return {
         model: cuaModel,
         reasoning: { effort: "medium" },
@@ -130,97 +101,23 @@ export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo
         store: true,
         truncation: "auto",
         tools: [
-            {
-                type: "computer",
-            },
+            { type: "computer" },
             previewClickTool,
         ],
     };
 }
-function shouldMapPreviewAccessError(err) {
-    const status = err?.status;
-    const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
-    const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
-    const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
-    if (status === 403 || status === 404)
-        return true;
-    if (code.includes("model_not_found") || code.includes("permission"))
-        return true;
-    if (type.includes("permission"))
-        return true;
-    return (message.includes("computer-use-preview") &&
-        (message.includes("access") || message.includes("permission") || message.includes("not found") || message.includes("unsupported")));
-}
-export function shouldMapMiniCompatibilityError(err) {
-    const status = Number(err?.status);
-    const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
-    const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
-    const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
-    const mentionsMiniModel = message.includes("gpt-5.4-mini");
-    const mentionsComputerTool = message.includes("computer") || message.includes("tool");
-    const mentionsUnsupported = (message.includes("unsupported") ||
-        message.includes("not supported") ||
-        message.includes("does not support"));
-    if (code.includes("model_not_found")) {
-        return true;
-    }
-    if (status === 404) {
-        return mentionsMiniModel && (message.includes("not found") || mentionsUnsupported);
-    }
-    if (code.includes("unsupported")) {
-        return mentionsMiniModel || mentionsComputerTool;
-    }
-    if (!type.includes("invalid")) {
-        return false;
-    }
-    return mentionsUnsupported && (mentionsMiniModel || mentionsComputerTool);
-}
-function mapCuaError(err, cuaModel) {
-    if (cuaModel === "computer-use-preview" && shouldMapPreviewAccessError(err)) {
-        const mapped = new Error("OpenAI API key does not have access to computer-use-preview. Switch to gpt-5.4 in Settings > CUA Model.");
-        mapped.cause = err;
-        return mapped;
-    }
-    if (cuaModel === "gpt-5.4-mini" && shouldMapMiniCompatibilityError(err)) {
-        const mapped = new Error("OpenAI rejected gpt-5.4-mini for computer-use execution. Switch to gpt-5.4 or computer-use-preview in Settings > CUA Model.");
-        mapped.cause = err;
-        return mapped;
-    }
-    return err;
-}
-export function isNonRetryableCuaError(err) {
-    const status = Number(err?.status);
-    const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
-    const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
-    const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
-    if ([401, 403, 404].includes(status)) {
-        return true;
-    }
-    return (code.includes("model_not_found") ||
-        code.includes("permission") ||
-        type.includes("permission") ||
-        message.includes("does not have access to computer-use-preview") ||
-        message.includes("switch to gpt-5.4 in settings > cua model") ||
-        message.includes("openai rejected gpt-5.4-mini for computer-use execution") ||
-        message.includes("switch to gpt-5.4 or computer-use-preview in settings > cua model"));
-}
-function getOpenAI() {
-    if (!openai) {
-        openai = new OpenAI({
-            apiKey: process.env.OPENAI_API_KEY,
-        });
-    }
-    return openai;
-}
 /**
  * Revise a test script based on user feedback using simple chat completion
- * @param {string} originalScript - The original test script
- * @param {string} revisionRequest - User's requested changes
+ * @param {{
+ *   client: import("openai").default,
+ *   originalScript: string,
+ *   revisionRequest: string,
+ * }} options
  * @returns {Promise<string>} - The revised test script
  */
-export async function reviseTestScript(originalScript, revisionRequest) {
-    const response = await getOpenAI().chat.completions.create({
-        model: "gpt-4o",
+export async function reviseTestScript({ client, originalScript, revisionRequest }) {
+    const response = await client.chat.completions.create({
+        model: "gpt-5.4",
         messages: [{
                 role: "system",
                 content: buildTestRevisionSystemPrompt(originalScript, revisionRequest)
@@ -228,29 +125,28 @@ export async function reviseTestScript(originalScript, revisionRequest) {
     });
     return response.choices[0].message.content.trim();
 }
-export async function compactAppContext({ contextDocument, taskDescription, tokenBudget }) {
-    const response = await getOpenAI().responses.create({
-        model: "gpt-5.4",
-        temperature: 0,
-        input: buildAppContextCompactionInput({
-            contextDocument,
-            taskDescription,
-            tokenBudget,
-        })
-    });
-    return {
-        briefing: typeof response.output_text === "string" ? response.output_text.trim() : "",
-        outputTokens: typeof response.usage?.output_tokens === "number" ? response.usage.output_tokens : null,
-    };
-}
-export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
-    const cuaModel = getSelectedCuaModel();
+/**
+ * @param {{
+ *   client: import("openai").default,
+ *   cuaModel: string,
+ *   contextOptimizationEnabled?: boolean,
+ *   contextOptimizationThreshold?: number,
+ *   messages: Array<unknown>,
+ *   screenshotBase64?: string,
+ *   previousResponseId?: string | null,
+ *   callId?: string | null,
+ *   deviceInfo?: unknown,
+ *   debugContext?: unknown,
+ * }} options
+ */
+export async function sendCUARequest({ client, cuaModel, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
     const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
     const requestParams = buildCuaRequestParams({
         cuaModel,
         previousResponseId,
-        deviceInfo,
         input,
+        contextOptimizationEnabled,
+        contextOptimizationThreshold,
     });
     const trace = cuaDebugTracer.startTurn({
         requestParams,
@@ -262,12 +158,44 @@ export async function sendCUARequest({ messages, screenshotBase64, previousRespo
     });
     logger.debug("CUA Request:", trace.requestLog);
     try {
-        const response = await getOpenAI().responses.create(requestParams);
+        const response = await client.responses.create(requestParams);
         cuaDebugTracer.onResponse(trace, response);
         return response;
     }
     catch (err) {
         cuaDebugTracer.onError(trace, err);
-        throw mapCuaError(err, cuaModel);
+        throw err;
     }
 }
+/**
+ * @param {{
+ *   client?: import("openai").default | null,
+ *   config?: Record<string, unknown>,
+ * }} options
+ */
+export function createOpenAiExecutor({ client, config }) {
+    const runtimeConfig = resolveRuntimeConfig(config);
+    const openAiClient = client || createOpenAiClient(null);
+    return {
+        client: openAiClient,
+        config: runtimeConfig,
+        async sendCUARequest(args) {
+            // Bind model and request-shaping config to the active run so execution
+            // stays deterministic even if settings change after the run starts.
+            return await sendCUARequest({
+                client: openAiClient,
+                cuaModel: runtimeConfig.cuaModel,
+                contextOptimizationEnabled: runtimeConfig.contextOptimizationEnabled,
+                contextOptimizationThreshold: runtimeConfig.contextOptimizationThreshold,
+                ...args,
+            });
+        },
+        async reviseTestScript(originalScript, revisionRequest) {
+            return await reviseTestScript({
+                client: openAiClient,
+                originalScript,
+                revisionRequest,
+            });
+        },
+    };
+}