@loadmill/droid-cua 2.4.0 → 2.6.0

package/build/src/device/assertions.js

@@ -35,7 +35,7 @@ export function extractFailureDetails(transcript) {
     const parts = recentTranscript.split("ASSERTION RESULT: FAIL");
     return parts[1]?.trim() || "Could not confidently validate the assertion.";
 }
-export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context, stepContext = null) {
+export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context, stepContext = null, eventPayload = {}) {
     const details = extractFailureDetails(transcript);
     const addOutput = context?.addOutput || printCliOutput;
     const meta = {
@@ -46,7 +46,8 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
         payload: {
             assertion: assertionPrompt,
             passed: false,
-            details
+            details,
+            ...eventPayload
         }
     };
     addOutput({ type: 'error', text: '❌ ASSERTION FAILED', ...meta });
@@ -61,7 +62,7 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
     }
     // Interactive mode: caller should clear remaining instructions
 }
-export function handleAssertionSuccess(assertionPrompt, context = null, stepContext = null) {
+export function handleAssertionSuccess(assertionPrompt, context = null, stepContext = null, eventPayload = {}) {
     const addOutput = context?.addOutput || printCliOutput;
     addOutput({
         type: 'success',
@@ -72,7 +73,8 @@ export function handleAssertionSuccess(assertionPrompt, context = null, stepCont
         instructionIndex: stepContext?.instructionIndex,
         payload: {
             assertion: assertionPrompt,
-            passed: true
+            passed: true,
+            ...eventPayload
         }
     });
 }

package/build/src/device/cloud/actions.js

@@ -2,6 +2,7 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 function normalizeMobileKeypress(platform, keys = []) {
     if (!Array.isArray(keys) || keys.length === 0) {
         throw new Error("Keypress action is missing keys");
@@ -53,6 +54,9 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform,
             source: session?.providerId,
             ...payload
@@ -83,7 +87,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             case "double_click": {
                 const targetX = Math.round((action.x / scale) / dpr);
                 const targetY = Math.round((action.y / scale) / dpr);
-                addOutput({ type: "action", text: `Tapping at (${targetX}, ${targetY})`, ...meta({ x: targetX, y: targetY }) });
+                addOutput({ type: "action", text: "Click", ...meta({ x: targetX, y: targetY }) });
                 await session.client.tap(session.sessionId, targetX, targetY);
                 break;
             }
@@ -93,14 +97,18 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 break;
             }
             case "scroll": {
-                const scrollX = Math.round((action.scroll_x / scale) / dpr);
-                const scrollY = Math.round((action.scroll_y / scale) / dpr);
-                const centerX = 200;
-                const centerY = 400;
-                const endX = centerX + scrollX;
-                const endY = centerY - scrollY;
-                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY }) });
-                await session.client.scroll(session.sessionId, centerX, centerY, endX, endY);
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    dpr,
+                    fallbackStartX: 200,
+                    fallbackStartY: 400
+                });
+                addOutput({
+                    type: "action",
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY})`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? "action" : "fallback" })
+                });
+                await session.client.scroll(session.sessionId, startX, startY, endX, endY);
                 break;
             }
             case "drag": {
@@ -176,6 +184,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform,
                 source: session.providerId

package/build/src/device/ios/actions.js

@@ -8,6 +8,7 @@ import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 function normalizeMobileKeypress(keys = []) {
     if (!Array.isArray(keys) || keys.length === 0) {
         throw new Error("Keypress action is missing keys");
@@ -50,6 +51,9 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform: "ios",
             ...payload
         }
@@ -81,7 +85,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 const pixelY = Math.round(action.y / scale);
                 const pointX = Math.round(pixelX / dpr);
                 const pointY = Math.round(pixelY / dpr);
-                addOutput({ type: "action", text: `Tapping at (${pointX}, ${pointY}) points`, ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
+                addOutput({ type: "action", text: "Click", ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
                 await appium.tap(session.sessionId, pointX, pointY);
                 break;
             }
@@ -92,15 +96,18 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             }
             case "scroll": {
                 const dpr = getDevicePixelRatio();
-                const scrollX = Math.round((action.scroll_x / scale) / dpr);
-                const scrollY = Math.round((action.scroll_y / scale) / dpr);
-                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY}) points`, ...meta({ scrollX, scrollY, unit: "points" }) });
-                // Start from center of screen (in logical points)
-                const centerX = 197; // Center of iPhone 16 (393/2)
-                const centerY = 426; // Center of iPhone 16 (852/2)
-                const endX = centerX + scrollX;
-                const endY = centerY - scrollY; // Invert Y for natural scrolling
-                await appium.scroll(session.sessionId, centerX, centerY, endX, endY);
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    dpr,
+                    fallbackStartX: 197,
+                    fallbackStartY: 426
+                });
+                addOutput({
+                    type: "action",
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY}) points`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? "action" : "fallback", unit: "points" })
+                });
+                await appium.scroll(session.sessionId, startX, startY, endX, endY);
                 break;
             }
             case "drag": {
@@ -176,6 +183,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform: "ios"
             }

package/build/src/device/openai.js

@@ -8,47 +8,87 @@ const cuaDebugTracer = new CuaDebugTracer(logger);
 function getSelectedCuaModel() {
     return process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
 }
-function normalizePreviewMessages(messages, screenshotBase64, includeInitialScreenshot) {
-    const normalized = messages.map((message) => {
-        if (!message?.role) {
-            return message;
-        }
-        const contentItems = Array.isArray(message.content)
-            ? message.content.map((item) => {
-                if (typeof item === "string") {
-                    return { type: "input_text", text: item };
-                }
-                return item;
-            })
-            : [{ type: "input_text", text: String(message.content ?? "") }];
-        return {
-            role: message.role,
-            content: contentItems,
-        };
-    });
-    if (!includeInitialScreenshot || !screenshotBase64) {
-        return normalized;
+function normalizeMessageContent(content) {
+    if (Array.isArray(content)) {
+        return content.map((item) => {
+            if (typeof item === "string") {
+                return { type: "input_text", text: item };
+            }
+            return item;
+        });
     }
-    for (let index = normalized.length - 1; index >= 0; index -= 1) {
-        const item = normalized[index];
+    return [{ type: "input_text", text: String(content ?? "") }];
+}
+function buildScreenshotDataUrl(screenshotBase64) {
+    return `data:image/png;base64,${screenshotBase64}`;
+}
+const previewClickTool = {
+    type: "function",
+    name: "preview_click",
+    description: "Preview where a proposed click would land on the current device screen. Returns an annotated screenshot with a marker at the given coordinates; the device state is unchanged and the click is not executed. Use this before issuing a click action to verify your aim.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            x: { type: "number" },
+            y: { type: "number" },
+        },
+        required: ["x", "y"],
+        additionalProperties: false,
+    },
+};
+function appendInitialScreenshot(input, screenshotBase64) {
+    // Most callers end with the active user message, but some recovery flows can
+    // reset the conversation to system-only state before the next screenshot send.
+    // Attach to the last user message when present, otherwise synthesize one below.
+    for (let index = input.length - 1; index >= 0; index -= 1) {
+        const item = input[index];
         if (item?.role !== "user" || !Array.isArray(item.content))
             continue;
         item.content.push({
             type: "input_image",
-            image_url: `data:image/png;base64,${screenshotBase64}`,
+            image_url: buildScreenshotDataUrl(screenshotBase64),
         });
-        return normalized;
+        return;
     }
-    normalized.push({
+    input.push({
         role: "user",
         content: [{
                 type: "input_image",
-                image_url: `data:image/png;base64,${screenshotBase64}`,
+                image_url: buildScreenshotDataUrl(screenshotBase64),
             }],
     });
-    return normalized;
 }
-function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
+export function buildCuaInput({ messages, screenshotBase64, previousResponseId, callId }) {
+    const input = messages.map((message) => {
+        if (!message?.role) {
+            return message;
+        }
+        return {
+            role: message.role,
+            content: normalizeMessageContent(message.content),
+        };
+    });
+    if (!screenshotBase64) {
+        return input;
+    }
+    if (callId) {
+        input.push({
+            type: "computer_call_output",
+            call_id: callId,
+            output: {
+                type: "computer_screenshot",
+                image_url: buildScreenshotDataUrl(screenshotBase64),
+            },
+        });
+        return input;
+    }
+    if (!previousResponseId) {
+        appendInitialScreenshot(input, screenshotBase64);
+    }
+    return input;
+}
+export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
     if (cuaModel === "computer-use-preview") {
         return {
             model: cuaModel,
@@ -56,24 +96,31 @@ function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input
             input,
             store: true,
             truncation: "auto",
-            tools: [{
+            tools: [
+                {
                     type: "computer_use_preview",
                     display_width: deviceInfo.scaled_width,
                     display_height: deviceInfo.scaled_height,
                     environment: "browser",
-                }],
+                },
+                previewClickTool,
+            ],
             reasoning: { summary: "concise" },
         };
     }
     return {
         model: cuaModel,
+        reasoning: { effort: "medium" },
         previous_response_id: previousResponseId || undefined,
         input,
         store: true,
         truncation: "auto",
-        tools: [{
+        tools: [
+            {
                 type: "computer",
-            }],
+            },
+            previewClickTool,
+        ],
     };
 }
 function shouldMapPreviewAccessError(err) {
@@ -153,20 +200,7 @@ export async function compactAppContext({ contextDocument, taskDescription, toke
 }
 export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
     const cuaModel = getSelectedCuaModel();
-    const includeInitialScreenshot = cuaModel === "computer-use-preview" && !previousResponseId && !callId;
-    const input = cuaModel === "computer-use-preview"
-        ? normalizePreviewMessages(messages, screenshotBase64, includeInitialScreenshot)
-        : [...messages];
-    if (callId && screenshotBase64) {
-        input.push({
-            type: "computer_call_output",
-            call_id: callId,
-            output: {
-                type: "computer_screenshot",
-                image_url: `data:image/png;base64,${screenshotBase64}`,
-            },
-        });
-    }
+    const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
     const requestParams = buildCuaRequestParams({
         cuaModel,
         previousResponseId,

package/build/src/device/screenshot-resolution.js

@@ -0,0 +1,33 @@
+export const TARGET_SCALED_WIDTH = 400;
+export const SCREENSHOT_RESOLUTION_MODE_DOWNSCALED = "downscaled";
+export const SCREENSHOT_RESOLUTION_MODE_NATIVE = "native";
+export function normalizeScreenshotResolutionMode(value) {
+    return value === SCREENSHOT_RESOLUTION_MODE_NATIVE
+        ? SCREENSHOT_RESOLUTION_MODE_NATIVE
+        : SCREENSHOT_RESOLUTION_MODE_DOWNSCALED;
+}
+export function validateScreenshotResolutionMode(value, label) {
+    if (typeof value !== "string") {
+        throw new Error(`${label} must be one of: downscaled, native.`);
+    }
+    const normalized = normalizeScreenshotResolutionMode(value);
+    if (normalized !== value) {
+        throw new Error(`${label} must be one of: downscaled, native.`);
+    }
+    return normalized;
+}
+export function buildResolutionAwareDeviceInfo({ width, height, screenshotResolutionMode, }) {
+    const normalizedMode = normalizeScreenshotResolutionMode(screenshotResolutionMode);
+    const scale = normalizedMode === SCREENSHOT_RESOLUTION_MODE_NATIVE || width <= TARGET_SCALED_WIDTH
+        ? 1.0
+        : TARGET_SCALED_WIDTH / width;
+    return {
+        scaled_width: Math.round(width * scale),
+        scaled_height: Math.round(height * scale),
+        scale,
+        screenshot_resolution_mode: normalizedMode,
+    };
+}
+export function readScreenshotResolutionModeFromEnv() {
+    return normalizeScreenshotResolutionMode(process.env.DROID_CUA_SCREENSHOT_RESOLUTION_MODE);
+}

package/build/src/device/scroll-gesture.js

@@ -0,0 +1,20 @@
+// The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
+// anchor coordinates `x` / `y`. This helper converts those model coordinates
+// into backend gesture coordinates: startX/startY and endX/endY.
+export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
+    const divisor = scale * dpr;
+    const scrollX = Math.round((action?.scroll_x ?? 0) / divisor);
+    const scrollY = Math.round((action?.scroll_y ?? 0) / divisor);
+    const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
+    const startX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
+    const startY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
+    return {
+        scrollX,
+        scrollY,
+        startX,
+        startY,
+        endX: startX + scrollX,
+        endY: startY - scrollY,
+        hasAnchor
+    };
+}

package/build/src/modes/execution-mode.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
 import { sendCUARequest } from "../device/openai.js";
 import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
@@ -14,6 +15,22 @@ export function buildExecutionRequestPayload({ instruction, isAssertionStep, mes
         previousResponseIdToSend: previousResponseId,
     };
 }
+function screenshotPathFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
+            ? result.screenshotPath
+            : null;
+    }
+    return null;
+}
+function screenshotDataUrlFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
+            ? result.screenshotDataUrl
+            : null;
+    }
+    return null;
+}
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -197,6 +214,7 @@ export class ExecutionMode {
         }
         // ── Check for assertion ──
         const isAssertionStep = isAssertion(instruction);
+        const assertionEventId = isAssertionStep ? randomUUID() : null;
         let assertionPrompt = null;
         const transcriptStartIndex = this.session.transcript.length;
         if (isAssertionStep) {
@@ -213,12 +231,27 @@ export class ExecutionMode {
         }
         try {
             const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-            await this.engine.recordScreenshot?.(screenshotBase64, {
+            const screenshotResult = await this.engine.recordScreenshot?.(screenshotBase64, {
                 runId: context?.runId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
+                ...(assertionEventId ? { eventId: assertionEventId } : {}),
                 captureSource: isAssertionStep ? "instruction-input-assertion" : "instruction-input"
             });
+            const assertionScreenshotPath = isAssertionStep ? screenshotPathFromRecorderResult(screenshotResult) : null;
+            const assertionScreenshotDataUrl = isAssertionStep
+                ? screenshotDataUrlFromRecorderResult(screenshotResult) ||
+                    (context?.includeScreenshotDataUrl && assertionScreenshotPath
+                        ? `data:image/png;base64,${screenshotBase64}`
+                        : null)
+                : null;
+            const assertionEventPayload = isAssertionStep
+                ? {
+                    eventId: assertionEventId,
+                    ...(assertionScreenshotPath ? { screenshotPath: assertionScreenshotPath } : {}),
+                    ...(assertionScreenshotDataUrl ? { screenshotDataUrl: assertionScreenshotDataUrl } : {})
+                }
+                : {};
             // Assertions rely on the prior response chain for earlier execution context.
             const { messagesToSend, previousResponseIdToSend } = buildExecutionRequestPayload({
                 instruction,
@@ -253,7 +286,7 @@ export class ExecutionMode {
                 const result = checkAssertionResult(assertionTranscript);
                 if (result.failed) {
                     handleAssertionFailure(assertionPrompt, assertionTranscript, false, // Never exit process - we'll always prompt the user in interactive mode
-                    context, stepContext);
+                    context, stepContext, assertionEventPayload);
                     // In headless mode, exit immediately on assertion failure
                     if (this.isHeadlessMode) {
                         this.stats.assertionsFailed++;
@@ -297,7 +330,7 @@ export class ExecutionMode {
                 }
                 else if (result.passed) {
                     this.stats.assertionsPassed++;
-                    handleAssertionSuccess(assertionPrompt, context, stepContext);
+                    handleAssertionSuccess(assertionPrompt, context, stepContext, assertionEventPayload);
                 }
             }
             // Clear messages after each turn but KEEP the response chain for context

package/build/src/prompts/base.js

@@ -62,6 +62,10 @@ function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
 
   When aiming for visual targets:
   - Reason carefully about the approximate pixel position.
+  - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
+  - Review the returned annotated screenshot before clicking.
+  - Only click if the marker is clearly inside the intended target and not on its edge, border, text, or nearby empty space.
+  - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
   - Click precisely based on your visual estimate.
 
   Available actions: click, scroll, type, keypress, wait, screenshot.

package/build/src/test-store/test-manager.js

@@ -1,5 +1,6 @@
 import { readdir, readFile, writeFile, unlink, stat, mkdir } from "fs/promises";
 import path from "path";
+import { countExecutableInstructions, parseTestScript } from "./test-script.js";
 // Tests directory is relative to current working directory
 const TESTS_DIR = path.join(process.cwd(), "tests");
 /**
@@ -27,10 +28,7 @@ export async function loadTest(name) {
     const filename = name.endsWith(".dcua") ? name : `${name}.dcua`;
     const filepath = path.join(TESTS_DIR, filename);
     const content = await readFile(filepath, "utf-8");
-    return content
-        .split("\n")
-        .map(line => line.trim())
-        .filter(line => line.length > 0);
+    return parseTestScript(content);
 }
 /**
  * Get the raw content of a test file
@@ -60,7 +58,7 @@ export async function listTests() {
         const filepath = path.join(TESTS_DIR, filename);
         const stats = await stat(filepath);
         const content = await readFile(filepath, "utf-8");
-        const lines = content.split("\n").filter(line => line.trim().length > 0).length;
+        const lines = countExecutableInstructions(content);
         return {
             name: filename.replace(".dcua", ""),
             filename: filename,

package/build/src/test-store/test-script.js

@@ -0,0 +1,50 @@
+/**
+ * Find the first comment marker in a test script line.
+ * `//` starts a comment only at line start or when preceded by whitespace.
+ *
+ * @param {string} line
+ * @returns {number}
+ */
+export function findCommentStart(line) {
+    for (let i = 0; i < line.length - 1; i++) {
+        if (line[i] !== "/" || line[i + 1] !== "/") {
+            continue;
+        }
+        if (i === 0 || /\s/.test(line[i - 1])) {
+            return i;
+        }
+    }
+    return -1;
+}
+/**
+ * Remove inline comments and surrounding instruction whitespace from a line.
+ *
+ * @param {string} line
+ * @returns {string}
+ */
+export function stripInstructionComment(line) {
+    const commentStart = findCommentStart(line);
+    const instruction = commentStart >= 0 ? line.slice(0, commentStart) : line;
+    return instruction.trim();
+}
+/**
+ * Parse executable instructions from raw .dcua content.
+ *
+ * @param {string} content
+ * @returns {string[]}
+ */
+export function parseTestScript(content) {
+    return content
+        .split("\n")
+        .map(stripInstructionComment)
+        .filter((line) => line.length > 0);
+}
+/**
+ * Count executable instructions in raw .dcua content.
+ *
+ * @param {string} content
+ * @returns {number}
+ */
+export function countExecutableInstructions(content) {
+    return parseTestScript(content).length;
+}

package/build/src/utils/cua-debug-tracer.js

@@ -63,6 +63,19 @@ function summarizeRequestInput(input) {
                 hasSafetyChecks: Array.isArray(item.acknowledged_safety_checks) && item.acknowledged_safety_checks.length > 0
             };
         }
+        if (item?.type === "function_call_output") {
+            const content = Array.isArray(item.output) ? item.output : [];
+            const imageBytes = content
+                .filter((entry) => entry?.type === "input_image" && typeof entry.image_url === "string")
+                .map((entry) => entry.image_url.replace("data:image/png;base64,", "").length);
+            return {
+                type: "function_call_output",
+                callId: item.call_id,
+                contentTypes: content.map((entry) => entry?.type).filter(Boolean),
+                imageBytes,
+                text: summarizeContent(content),
+            };
+        }
         return {
             type: item?.type || "unknown"
         };
@@ -85,6 +98,21 @@ function redactRequestForFileLog(requestParams) {
                     acknowledged_safety_checks: item.acknowledged_safety_checks
                 };
             }
+            if (item.type === "function_call_output" && Array.isArray(item.output)) {
+                return {
+                    ...item,
+                    output: item.output.map((entry) => {
+                        if (entry?.type === "input_image" && typeof entry.image_url === "string") {
+                            const base64Data = entry.image_url.replace("data:image/png;base64,", "");
+                            return {
+                                ...entry,
+                                image_url: `data:image/png;base64,[${base64Data.length} chars]`
+                            };
+                        }
+                        return entry;
+                    })
+                };
+            }
             return item;
         })
     };
@@ -117,6 +145,15 @@ function summarizeResponseOutput(response) {
                 actionType: item.action?.type
             };
         }
+        if (item.type === "function_call") {
+            return {
+                type: "function_call",
+                id: item.id,
+                callId: item.call_id,
+                name: item.name,
+                arguments: truncateForDebug(item.arguments, 300)
+            };
+        }
         if (item.type === "pending_safety_check") {
             return { type: "pending_safety_check", id: item.id, code: item.code };
         }
@@ -175,11 +212,11 @@ export class CuaDebugTracer {
             }));
         });
         const inputCallIds = input
-            .filter((item) => item?.type === "computer_call_output")
+            .filter((item) => item?.type === "computer_call_output" || item?.type === "function_call_output")
             .map((item) => item.call_id)
             .filter(Boolean);
         const inputItemTypes = input.map((item) => item?.type || (item?.role ? `message:${item.role}` : "unknown"));
-        const lastResponseCallIds = Array.isArray(lastResponseMeta?.computerCallIds) ? lastResponseMeta.computerCallIds : [];
+        const lastResponseCallIds = Array.isArray(lastResponseMeta?.toolCallIds) ? lastResponseMeta.toolCallIds : [];
         const missingCallIds = inputCallIds.filter((callId) => !lastResponseCallIds.includes(callId));
         const previousResponseIdMatchesLastResponseId = !previousResponseId || !lastResponseMeta?.id ? null : previousResponseId === lastResponseMeta.id;
         const allInputCallIdsFoundInLastResponse = inputCallIds.length === 0 ? true : missingCallIds.length === 0;
@@ -252,10 +289,12 @@ export class CuaDebugTracer {
     onResponse(trace, response) {
         const outputTypes = (response.output || []).map((item) => item.type);
         const toolCalls = (response.output || [])
-            .filter((item) => item.type === "computer_call")
+            .filter((item) => item.type === "computer_call" || item.type === "function_call")
             .map((item) => ({
             call_id: item.call_id,
-            action_type: item.action?.type
+            action_type: item.action?.type,
+            name: item.name,
+            type: item.type
         }));
         const safetyChecks = (response.output || [])
             .filter((item) => item.type === "pending_safety_check")
@@ -268,7 +307,11 @@ export class CuaDebugTracer {
         this.lastResponseMetaByChain.set(trace.chainKey, {
             id: response.id,
             outputTypes,
-            computerCallIds: toolCalls.map((item) => item.call_id).filter(Boolean),
+            toolCallIds: toolCalls.map((item) => item.call_id).filter(Boolean),
+            computerCallIds: toolCalls
+                .filter((item) => item.type === "computer_call")
+                .map((item) => item.call_id)
+                .filter(Boolean),
             pendingSafetyCheckIds: safetyChecks.map((item) => item.id).filter(Boolean)
         });
         const accountedItems = toolCalls.length + safetyChecks.length;