@loadmill/droid-cua 2.5.0 → 2.7.0

package/build/src/device/cloud/actions.js

@@ -54,6 +54,9 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform,
             source: session?.providerId,
             ...payload
@@ -84,7 +87,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             case "double_click": {
                 const targetX = Math.round((action.x / scale) / dpr);
                 const targetY = Math.round((action.y / scale) / dpr);
-                addOutput({ type: "action", text: `Tapping at (${targetX}, ${targetY})`, ...meta({ x: targetX, y: targetY }) });
+                addOutput({ type: "action", text: "Click", ...meta({ x: targetX, y: targetY }) });
                 await session.client.tap(session.sessionId, targetX, targetY);
                 break;
             }
@@ -181,6 +184,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform,
                 source: session.providerId

package/build/src/device/ios/actions.js

@@ -51,6 +51,9 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform: "ios",
             ...payload
         }
@@ -82,7 +85,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 const pixelY = Math.round(action.y / scale);
                 const pointX = Math.round(pixelX / dpr);
                 const pointY = Math.round(pixelY / dpr);
-                addOutput({ type: "action", text: `Tapping at (${pointX}, ${pointY}) points`, ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
+                addOutput({ type: "action", text: "Click", ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
                 await appium.tap(session.sessionId, pointX, pointY);
                 break;
             }
@@ -180,6 +183,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform: "ios"
             }

package/build/src/device/ios/appium-server.js

@@ -7,6 +7,7 @@
 import { spawn } from "child_process";
 import { existsSync, readdirSync } from "fs";
 import path from "path";
+import { COMMON_NODE_BIN_DIRS, makeNodeCliCommandEnv } from "./node-cli-env.js";
 let appiumProcess = null;
 let cleanupRegistered = false;
 function getAppiumBaseUrl() {
@@ -35,8 +36,7 @@ export function resolveAppiumCommand() {
         return explicit;
     }
     const commonCandidates = [
-        "/opt/homebrew/bin/appium",
-        "/usr/local/bin/appium",
+        ...COMMON_NODE_BIN_DIRS.map((dir) => path.join(dir, "appium")),
         ...collectNvmAppiumCandidates()
     ];
     for (const candidate of commonCandidates) {
@@ -50,7 +50,8 @@ async function spawnAppiumProcess(command) {
     return await new Promise((resolve, reject) => {
         const child = spawn(command, [], {
             detached: true,
-            stdio: "ignore"
+            stdio: "ignore",
+            env: makeNodeCliCommandEnv(command)
         });
         child.once("error", reject);
         child.once("spawn", () => resolve(child));

package/build/src/device/ios/node-cli-env.js

@@ -0,0 +1,28 @@
+import path from "path";
+export const COMMON_NODE_BIN_DIRS = ["/opt/homebrew/bin", "/usr/local/bin", "/opt/local/bin"];
+function prependPathEntries(entries, originalPath) {
+    const seen = new Set();
+    const next = [];
+    const allEntries = [
+        ...entries,
+        ...(originalPath
+            ?.split(path.delimiter)
+            .map((entry) => entry.trim())
+            .filter(Boolean) ?? [])
+    ];
+    for (const entry of allEntries) {
+        if (seen.has(entry))
+            continue;
+        seen.add(entry);
+        next.push(entry);
+    }
+    return next.join(path.delimiter);
+}
+export function makeNodeCliCommandEnv(command) {
+    const binDirs = path.isAbsolute(command) ? [path.dirname(command), ...COMMON_NODE_BIN_DIRS] : COMMON_NODE_BIN_DIRS;
+    return {
+        ...process.env,
+        // Finder-launched apps may miss Homebrew paths; npm/Appium shims use "#!/usr/bin/env node".
+        PATH: prependPathEntries(binDirs, process.env.PATH)
+    };
+}

package/build/src/device/openai.js

@@ -1,54 +1,107 @@
 import OpenAI from "openai";
+import { readContextOptimizationEnv } from "../core/context-optimization.js";
 import { buildTestRevisionSystemPrompt } from "../prompts/editor.js";
 import { buildAppContextCompactionInput } from "../prompts/execution.js";
 import { logger } from "../utils/logger.js";
 import { CuaDebugTracer } from "../utils/cua-debug-tracer.js";
 let openai = null;
 const cuaDebugTracer = new CuaDebugTracer(logger);
+const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
 function getSelectedCuaModel() {
-    return process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
+    return VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL) ? process.env.OPENAI_CUA_MODEL : "gpt-5.4";
 }
-function normalizePreviewMessages(messages, screenshotBase64, includeInitialScreenshot) {
-    const normalized = messages.map((message) => {
-        if (!message?.role) {
-            return message;
-        }
-        const contentItems = Array.isArray(message.content)
-            ? message.content.map((item) => {
-                if (typeof item === "string") {
-                    return { type: "input_text", text: item };
-                }
-                return item;
-            })
-            : [{ type: "input_text", text: String(message.content ?? "") }];
-        return {
-            role: message.role,
-            content: contentItems,
-        };
-    });
-    if (!includeInitialScreenshot || !screenshotBase64) {
-        return normalized;
+function normalizeMessageContent(content) {
+    if (Array.isArray(content)) {
+        return content.map((item) => {
+            if (typeof item === "string") {
+                return { type: "input_text", text: item };
+            }
+            return item;
+        });
     }
-    for (let index = normalized.length - 1; index >= 0; index -= 1) {
-        const item = normalized[index];
+    return [{ type: "input_text", text: String(content ?? "") }];
+}
+function buildScreenshotDataUrl(screenshotBase64) {
+    return `data:image/png;base64,${screenshotBase64}`;
+}
+const previewClickTool = {
+    type: "function",
+    name: "preview_click",
+    description: "Preview where a proposed click would land on the current device screen. Returns an annotated screenshot with a marker at the given coordinates; the device state is unchanged and the click is not executed. Use this before issuing a click action to verify your aim.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            x: { type: "number" },
+            y: { type: "number" },
+        },
+        required: ["x", "y"],
+        additionalProperties: false,
+    },
+};
+function appendInitialScreenshot(input, screenshotBase64) {
+    // Most callers end with the active user message, but some recovery flows can
+    // reset the conversation to system-only state before the next screenshot send.
+    // Attach to the last user message when present, otherwise synthesize one below.
+    for (let index = input.length - 1; index >= 0; index -= 1) {
+        const item = input[index];
         if (item?.role !== "user" || !Array.isArray(item.content))
             continue;
         item.content.push({
             type: "input_image",
-            image_url: `data:image/png;base64,${screenshotBase64}`,
+            image_url: buildScreenshotDataUrl(screenshotBase64),
         });
-        return normalized;
+        return;
     }
-    normalized.push({
+    input.push({
         role: "user",
         content: [{
                 type: "input_image",
-                image_url: `data:image/png;base64,${screenshotBase64}`,
+                image_url: buildScreenshotDataUrl(screenshotBase64),
             }],
     });
-    return normalized;
 }
-function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
+export function buildCuaInput({ messages, screenshotBase64, previousResponseId, callId }) {
+    const input = messages.map((message) => {
+        if (!message?.role) {
+            return message;
+        }
+        return {
+            role: message.role,
+            content: normalizeMessageContent(message.content),
+        };
+    });
+    if (!screenshotBase64) {
+        return input;
+    }
+    if (callId) {
+        input.push({
+            type: "computer_call_output",
+            call_id: callId,
+            output: {
+                type: "computer_screenshot",
+                image_url: buildScreenshotDataUrl(screenshotBase64),
+            },
+        });
+        return input;
+    }
+    if (!previousResponseId) {
+        appendInitialScreenshot(input, screenshotBase64);
+    }
+    return input;
+}
+function buildContextManagement(cuaModel) {
+    if (cuaModel === "computer-use-preview") {
+        return undefined;
+    }
+    const { enabled, threshold } = readContextOptimizationEnv();
+    if (!enabled) {
+        return undefined;
+    }
+    return [{ type: "compaction", compact_threshold: threshold }];
+}
+export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
+    const contextManagement = buildContextManagement(cuaModel);
     if (cuaModel === "computer-use-preview") {
         return {
             model: cuaModel,
@@ -56,24 +109,32 @@ function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input
             input,
             store: true,
             truncation: "auto",
-            tools: [{
+            tools: [
+                {
                     type: "computer_use_preview",
                     display_width: deviceInfo.scaled_width,
                     display_height: deviceInfo.scaled_height,
                     environment: "browser",
-                }],
+                },
+                previewClickTool,
+            ],
             reasoning: { summary: "concise" },
         };
     }
     return {
         model: cuaModel,
+        reasoning: { effort: "medium" },
+        ...(contextManagement ? { context_management: contextManagement } : {}),
         previous_response_id: previousResponseId || undefined,
         input,
         store: true,
         truncation: "auto",
-        tools: [{
+        tools: [
+            {
                 type: "computer",
-            }],
+            },
+            previewClickTool,
+        ],
     };
 }
 function shouldMapPreviewAccessError(err) {
@@ -90,12 +151,41 @@ function shouldMapPreviewAccessError(err) {
     return (message.includes("computer-use-preview") &&
         (message.includes("access") || message.includes("permission") || message.includes("not found") || message.includes("unsupported")));
 }
+export function shouldMapMiniCompatibilityError(err) {
+    const status = Number(err?.status);
+    const code = typeof err?.code === "string" ? err.code.toLowerCase() : "";
+    const type = typeof err?.type === "string" ? err.type.toLowerCase() : "";
+    const message = typeof err?.message === "string" ? err.message.toLowerCase() : "";
+    const mentionsMiniModel = message.includes("gpt-5.4-mini");
+    const mentionsComputerTool = message.includes("computer") || message.includes("tool");
+    const mentionsUnsupported = (message.includes("unsupported") ||
+        message.includes("not supported") ||
+        message.includes("does not support"));
+    if (code.includes("model_not_found")) {
+        return true;
+    }
+    if (status === 404) {
+        return mentionsMiniModel && (message.includes("not found") || mentionsUnsupported);
+    }
+    if (code.includes("unsupported")) {
+        return mentionsMiniModel || mentionsComputerTool;
+    }
+    if (!type.includes("invalid")) {
+        return false;
+    }
+    return mentionsUnsupported && (mentionsMiniModel || mentionsComputerTool);
+}
 function mapCuaError(err, cuaModel) {
     if (cuaModel === "computer-use-preview" && shouldMapPreviewAccessError(err)) {
         const mapped = new Error("OpenAI API key does not have access to computer-use-preview. Switch to gpt-5.4 in Settings > CUA Model.");
         mapped.cause = err;
         return mapped;
     }
+    if (cuaModel === "gpt-5.4-mini" && shouldMapMiniCompatibilityError(err)) {
+        const mapped = new Error("OpenAI rejected gpt-5.4-mini for computer-use execution. Switch to gpt-5.4 or computer-use-preview in Settings > CUA Model.");
+        mapped.cause = err;
+        return mapped;
+    }
     return err;
 }
 export function isNonRetryableCuaError(err) {
@@ -110,7 +200,9 @@ export function isNonRetryableCuaError(err) {
         code.includes("permission") ||
         type.includes("permission") ||
         message.includes("does not have access to computer-use-preview") ||
-        message.includes("switch to gpt-5.4 in settings > cua model"));
+        message.includes("switch to gpt-5.4 in settings > cua model") ||
+        message.includes("openai rejected gpt-5.4-mini for computer-use execution") ||
+        message.includes("switch to gpt-5.4 or computer-use-preview in settings > cua model"));
 }
 function getOpenAI() {
     if (!openai) {
@@ -153,20 +245,7 @@ export async function compactAppContext({ contextDocument, taskDescription, toke
 }
 export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
     const cuaModel = getSelectedCuaModel();
-    const includeInitialScreenshot = cuaModel === "computer-use-preview" && !previousResponseId && !callId;
-    const input = cuaModel === "computer-use-preview"
-        ? normalizePreviewMessages(messages, screenshotBase64, includeInitialScreenshot)
-        : [...messages];
-    if (callId && screenshotBase64) {
-        input.push({
-            type: "computer_call_output",
-            call_id: callId,
-            output: {
-                type: "computer_screenshot",
-                image_url: `data:image/png;base64,${screenshotBase64}`,
-            },
-        });
-    }
+    const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
     const requestParams = buildCuaRequestParams({
         cuaModel,
         previousResponseId,

package/build/src/modes/execution-mode.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
 import { sendCUARequest } from "../device/openai.js";
 import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
@@ -14,6 +15,22 @@ export function buildExecutionRequestPayload({ instruction, isAssertionStep, mes
         previousResponseIdToSend: previousResponseId,
     };
 }
+function screenshotPathFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
+            ? result.screenshotPath
+            : null;
+    }
+    return null;
+}
+function screenshotDataUrlFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
+            ? result.screenshotDataUrl
+            : null;
+    }
+    return null;
+}
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -197,6 +214,7 @@ export class ExecutionMode {
         }
         // ── Check for assertion ──
         const isAssertionStep = isAssertion(instruction);
+        const assertionEventId = isAssertionStep ? randomUUID() : null;
         let assertionPrompt = null;
         const transcriptStartIndex = this.session.transcript.length;
         if (isAssertionStep) {
@@ -213,12 +231,27 @@ export class ExecutionMode {
         }
         try {
             const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-            await this.engine.recordScreenshot?.(screenshotBase64, {
+            const screenshotResult = await this.engine.recordScreenshot?.(screenshotBase64, {
                 runId: context?.runId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
+                ...(assertionEventId ? { eventId: assertionEventId } : {}),
                 captureSource: isAssertionStep ? "instruction-input-assertion" : "instruction-input"
             });
+            const assertionScreenshotPath = isAssertionStep ? screenshotPathFromRecorderResult(screenshotResult) : null;
+            const assertionScreenshotDataUrl = isAssertionStep
+                ? screenshotDataUrlFromRecorderResult(screenshotResult) ||
+                    (context?.includeScreenshotDataUrl && assertionScreenshotPath
+                        ? `data:image/png;base64,${screenshotBase64}`
+                        : null)
+                : null;
+            const assertionEventPayload = isAssertionStep
+                ? {
+                    eventId: assertionEventId,
+                    ...(assertionScreenshotPath ? { screenshotPath: assertionScreenshotPath } : {}),
+                    ...(assertionScreenshotDataUrl ? { screenshotDataUrl: assertionScreenshotDataUrl } : {})
+                }
+                : {};
             // Assertions rely on the prior response chain for earlier execution context.
             const { messagesToSend, previousResponseIdToSend } = buildExecutionRequestPayload({
                 instruction,
@@ -253,7 +286,7 @@ export class ExecutionMode {
                 const result = checkAssertionResult(assertionTranscript);
                 if (result.failed) {
                     handleAssertionFailure(assertionPrompt, assertionTranscript, false, // Never exit process - we'll always prompt the user in interactive mode
-                    context, stepContext);
+                    context, stepContext, assertionEventPayload);
                     // In headless mode, exit immediately on assertion failure
                     if (this.isHeadlessMode) {
                         this.stats.assertionsFailed++;
@@ -297,7 +330,7 @@ export class ExecutionMode {
                 }
                 else if (result.passed) {
                     this.stats.assertionsPassed++;
-                    handleAssertionSuccess(assertionPrompt, context, stepContext);
+                    handleAssertionSuccess(assertionPrompt, context, stepContext, assertionEventPayload);
                 }
             }
             // Clear messages after each turn but KEEP the response chain for context

package/build/src/prompts/base.js

@@ -62,6 +62,12 @@ function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
 
   When aiming for visual targets:
   - Reason carefully about the approximate pixel position.
+  - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
+  - Review the returned annotated screenshot before clicking.
+  - Click if the marker is clearly inside the intended tappable control. Text inside a button, field, row, or bottom bar is okay.
+  - Adjust and preview again only if the marker is outside the target, on the target border/edge, or closer to a neighboring target than the intended one.
+  - If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
+  - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
   - Click precisely based on your visual estimate.
 
   Available actions: click, scroll, type, keypress, wait, screenshot.

package/build/src/utils/cua-debug-tracer.js

@@ -63,6 +63,19 @@ function summarizeRequestInput(input) {
                 hasSafetyChecks: Array.isArray(item.acknowledged_safety_checks) && item.acknowledged_safety_checks.length > 0
             };
         }
+        if (item?.type === "function_call_output") {
+            const content = Array.isArray(item.output) ? item.output : [];
+            const imageBytes = content
+                .filter((entry) => entry?.type === "input_image" && typeof entry.image_url === "string")
+                .map((entry) => entry.image_url.replace("data:image/png;base64,", "").length);
+            return {
+                type: "function_call_output",
+                callId: item.call_id,
+                contentTypes: content.map((entry) => entry?.type).filter(Boolean),
+                imageBytes,
+                text: summarizeContent(content),
+            };
+        }
         return {
             type: item?.type || "unknown"
         };
@@ -85,6 +98,21 @@ function redactRequestForFileLog(requestParams) {
                     acknowledged_safety_checks: item.acknowledged_safety_checks
                 };
             }
+            if (item.type === "function_call_output" && Array.isArray(item.output)) {
+                return {
+                    ...item,
+                    output: item.output.map((entry) => {
+                        if (entry?.type === "input_image" && typeof entry.image_url === "string") {
+                            const base64Data = entry.image_url.replace("data:image/png;base64,", "");
+                            return {
+                                ...entry,
+                                image_url: `data:image/png;base64,[${base64Data.length} chars]`
+                            };
+                        }
+                        return entry;
+                    })
+                };
+            }
             return item;
         })
     };
@@ -117,6 +145,15 @@ function summarizeResponseOutput(response) {
                 actionType: item.action?.type
             };
         }
+        if (item.type === "function_call") {
+            return {
+                type: "function_call",
+                id: item.id,
+                callId: item.call_id,
+                name: item.name,
+                arguments: truncateForDebug(item.arguments, 300)
+            };
+        }
         if (item.type === "pending_safety_check") {
             return { type: "pending_safety_check", id: item.id, code: item.code };
         }
@@ -175,11 +212,11 @@ export class CuaDebugTracer {
             }));
         });
         const inputCallIds = input
-            .filter((item) => item?.type === "computer_call_output")
+            .filter((item) => item?.type === "computer_call_output" || item?.type === "function_call_output")
             .map((item) => item.call_id)
             .filter(Boolean);
         const inputItemTypes = input.map((item) => item?.type || (item?.role ? `message:${item.role}` : "unknown"));
-        const lastResponseCallIds = Array.isArray(lastResponseMeta?.computerCallIds) ? lastResponseMeta.computerCallIds : [];
+        const lastResponseCallIds = Array.isArray(lastResponseMeta?.toolCallIds) ? lastResponseMeta.toolCallIds : [];
         const missingCallIds = inputCallIds.filter((callId) => !lastResponseCallIds.includes(callId));
         const previousResponseIdMatchesLastResponseId = !previousResponseId || !lastResponseMeta?.id ? null : previousResponseId === lastResponseMeta.id;
         const allInputCallIdsFoundInLastResponse = inputCallIds.length === 0 ? true : missingCallIds.length === 0;
@@ -252,10 +289,12 @@ export class CuaDebugTracer {
     onResponse(trace, response) {
         const outputTypes = (response.output || []).map((item) => item.type);
         const toolCalls = (response.output || [])
-            .filter((item) => item.type === "computer_call")
+            .filter((item) => item.type === "computer_call" || item.type === "function_call")
             .map((item) => ({
             call_id: item.call_id,
-            action_type: item.action?.type
+            action_type: item.action?.type,
+            name: item.name,
+            type: item.type
         }));
         const safetyChecks = (response.output || [])
             .filter((item) => item.type === "pending_safety_check")
@@ -268,7 +307,11 @@ export class CuaDebugTracer {
         this.lastResponseMetaByChain.set(trace.chainKey, {
             id: response.id,
             outputTypes,
-            computerCallIds: toolCalls.map((item) => item.call_id).filter(Boolean),
+            toolCallIds: toolCalls.map((item) => item.call_id).filter(Boolean),
+            computerCallIds: toolCalls
+                .filter((item) => item.type === "computer_call")
+                .map((item) => item.call_id)
+                .filter(Boolean),
             pendingSafetyCheckIds: safetyChecks.map((item) => item.id).filter(Boolean)
         });
         const accountedItems = toolCalls.length + safetyChecks.length;