@loadmill/droid-cua 2.5.0 → 2.6.0

package/build/src/core/execution-engine.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import path from "path";
 import { writeFile } from "fs/promises";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
@@ -6,31 +7,99 @@ import { sendCUARequest } from "../device/openai.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { printCliOutput } from "../utils/console-output.js";
-function extractComputerCalls(items) {
+import { annotateScreenshotArtifact, createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
+function extractActionableOutputItems(items) {
     const entries = [];
     for (const item of items) {
-        if (item?.type !== "computer_call")
-            continue;
-        const baseCallId = item.call_id || item.id;
-        if (!baseCallId)
+        if (item?.type === "function_call" && item.name === "preview_click") {
+            entries.push({
+                type: "preview_click",
+                call_id: item.call_id || item.id,
+                arguments: item.arguments,
+            });
             continue;
-        const actions = [];
-        if (Array.isArray(item.actions)) {
-            for (const actionEntry of item.actions) {
-                if (!actionEntry)
-                    continue;
-                const action = actionEntry.action || actionEntry;
-                if (action?.type)
-                    actions.push(action);
-            }
         }
-        else if (item.action?.type) {
-            actions.push(item.action);
+        if (item?.type === "computer_call") {
+            const baseCallId = item.call_id || item.id;
+            if (!baseCallId)
+                continue;
+            const actions = [];
+            if (Array.isArray(item.actions)) {
+                for (const actionEntry of item.actions) {
+                    if (!actionEntry)
+                        continue;
+                    const action = actionEntry.action || actionEntry;
+                    if (action?.type)
+                        actions.push(action);
+                }
+            }
+            else if (item.action?.type) {
+                actions.push(item.action);
+            }
+            entries.push({ type: "computer_call", call_id: baseCallId, actions });
         }
-        entries.push({ call_id: baseCallId, actions });
     }
     return entries;
 }
+function buildPreviewClickValidationError(argumentsText) {
+    if (!argumentsText || typeof argumentsText !== "string") {
+        return "Invalid preview_click arguments: expected a JSON object with numeric x and y.";
+    }
+    return `Invalid preview_click arguments: ${argumentsText}`;
+}
+function parsePreviewClickArguments(argumentsText) {
+    let parsedArguments;
+    try {
+        parsedArguments = JSON.parse(argumentsText);
+    }
+    catch {
+        return {
+            ok: false,
+            message: buildPreviewClickValidationError("arguments could not be parsed as JSON; expected numeric x and y."),
+        };
+    }
+    const x = parsedArguments?.x;
+    const y = parsedArguments?.y;
+    if (!Number.isFinite(x) || !Number.isFinite(y)) {
+        return {
+            ok: false,
+            message: buildPreviewClickValidationError("x and y must both be numeric."),
+        };
+    }
+    return {
+        ok: true,
+        x,
+        y,
+    };
+}
+function buildFunctionCallOutputItem(callId, output) {
+    return {
+        type: "function_call_output",
+        call_id: callId,
+        output,
+    };
+}
+function buildPreviewOutputText({ requestedX, requestedY, renderedX, renderedY, wasClamped }) {
+    const baseText = `Marker placed at (${renderedX}, ${renderedY}) on the current screen. The click has not been executed.`;
+    if (!wasClamped) {
+        return baseText;
+    }
+    return `${baseText} Requested point (${requestedX}, ${requestedY}) was off-screen; marker rendered at (${renderedX}, ${renderedY}).`;
+}
+function buildPreviewOutputContent({ text, annotatedScreenshotBase64 = null }) {
+    const content = [{
+            type: "input_text",
+            text,
+        }];
+    if (annotatedScreenshotBase64) {
+        content.push({
+            type: "input_image",
+            image_url: `data:image/png;base64,${annotatedScreenshotBase64}`,
+            detail: "high",
+        });
+    }
+    return content;
+}
 function getScopeAndIds(context = null, stepContext = null) {
     const scope = context?.sessionId ? "design" : "execution";
     const ids = scope === "design"
@@ -80,6 +149,37 @@ function buildStrictModePlan(actions = [], strictMode = false) {
         runtimeNote
     };
 }
+function normalizeArtifactPaths(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        if (Array.isArray(result.annotationSourcePaths)) {
+            return normalizeArtifactPaths(result.annotationSourcePaths);
+        }
+        return normalizeArtifactPaths(result.artifactPaths);
+    }
+    if (Array.isArray(result)) {
+        return result.filter((value) => typeof value === "string" && value.trim());
+    }
+    if (typeof result === "string" && result.trim()) {
+        return [result];
+    }
+    return [];
+}
+function screenshotPathFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
+            ? result.screenshotPath
+            : null;
+    }
+    return typeof result === "string" && result.trim() ? result : null;
+}
+function screenshotDataUrlFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
+            ? result.screenshotDataUrl
+            : null;
+    }
+    return null;
+}
 export class ExecutionEngine {
     constructor(session, options = {}) {
         this.session = session;
@@ -89,26 +189,45 @@ export class ExecutionEngine {
         this.strictMode = options.strictMode === true;
         this.stepDelayMs = getConfiguredStepDelayMs();
         this.reportedScreenshotWriteError = false;
+        this.reportedScreenshotAnnotateError = false;
+        this.lastRecordedScreenshotPaths = [];
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
     }
-    async recordScreenshot(screenshotBase64, metadata = {}) {
+    async saveScreenshotArtifact(screenshotBase64, metadata = {}, options = {}) {
         if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
             return null;
         }
+        const updateLastRecordedScreenshotPaths = options.updateLastRecordedScreenshotPaths !== false;
+        const previousRecordedScreenshotPaths = updateLastRecordedScreenshotPaths
+            ? null
+            : [...this.lastRecordedScreenshotPaths];
         try {
             if (this.screenshotRecorder?.saveScreenshot) {
-                return await this.screenshotRecorder.saveScreenshot(screenshotBase64, metadata);
+                const result = await this.screenshotRecorder.saveScreenshot(screenshotBase64, metadata);
+                if (updateLastRecordedScreenshotPaths) {
+                    this.lastRecordedScreenshotPaths = normalizeArtifactPaths(result);
+                }
+                return result;
             }
             if (this.recordScreenshots && this.screenshotDir) {
                 const framePath = path.join(this.screenshotDir, `frame_${String(Date.now())}.png`);
                 await writeFile(framePath, Buffer.from(screenshotBase64, "base64"));
+                if (updateLastRecordedScreenshotPaths) {
+                    this.lastRecordedScreenshotPaths = [framePath];
+                }
                 return framePath;
             }
         }
         catch (error) {
+            if (updateLastRecordedScreenshotPaths) {
+                this.lastRecordedScreenshotPaths = [];
+            }
+            else {
+                this.lastRecordedScreenshotPaths = previousRecordedScreenshotPaths;
+            }
             if (!this.reportedScreenshotWriteError) {
                 this.reportedScreenshotWriteError = true;
                 const scope = typeof metadata.sessionId === "string" ? "design" : "execution";
@@ -131,6 +250,100 @@ export class ExecutionEngine {
         }
         return null;
     }
+    async recordScreenshot(screenshotBase64, metadata = {}) {
+        return await this.saveScreenshotArtifact(screenshotBase64, metadata);
+    }
+    async recordPreviewScreenshot(screenshotBase64, metadata = {}) {
+        return await this.saveScreenshotArtifact(screenshotBase64, metadata, {
+            updateLastRecordedScreenshotPaths: false,
+        });
+    }
+    async annotateLatestScreenshot(action, metadata = {}) {
+        if (!(action?.type === "click" || action?.type === "double_click") || this.lastRecordedScreenshotPaths.length === 0) {
+            return null;
+        }
+        try {
+            const marker = {
+                x: action.x,
+                y: action.y,
+                style: "click",
+            };
+            const sourcePath = this.lastRecordedScreenshotPaths.find((filePath) => typeof filePath === "string" && filePath.trim());
+            if (sourcePath && this.screenshotRecorder?.saveAnnotatedScreenshot) {
+                return await this.screenshotRecorder.saveAnnotatedScreenshot(sourcePath, marker, metadata);
+            }
+            await Promise.all(this.lastRecordedScreenshotPaths.map((filePath) => annotateScreenshotArtifact(filePath, marker)));
+            return null;
+        }
+        catch (error) {
+            if (!this.reportedScreenshotAnnotateError) {
+                this.reportedScreenshotAnnotateError = true;
+                const scope = typeof metadata.sessionId === "string" ? "design" : "execution";
+                const ids = scope === "design"
+                    ? {
+                        sessionId: metadata.sessionId,
+                        stepId: metadata.stepId,
+                        instructionIndex: metadata.instructionIndex
+                    }
+                    : {
+                        runId: metadata.runId,
+                        stepId: metadata.stepId,
+                        instructionIndex: metadata.instructionIndex
+                    };
+                emitDesktopDebug("run.screenshot_annotate.error", scope, ids, {
+                    actionType: action?.type ?? null,
+                    x: action?.x ?? null,
+                    y: action?.y ?? null,
+                    message: error instanceof Error ? error.message : "Failed to annotate screenshot"
+                });
+            }
+        }
+        return null;
+    }
+    async handlePreviewClick(callId, argumentsText, context = null, stepContext = null) {
+        const parsedArguments = parsePreviewClickArguments(argumentsText);
+        if (!parsedArguments.ok) {
+            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({ text: parsedArguments.message }));
+        }
+        try {
+            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+            const previewResult = await createAnnotatedScreenshotFromBase64(screenshotBase64, {
+                x: parsedArguments.x,
+                y: parsedArguments.y,
+                style: "preview",
+            });
+            if (!previewResult?.screenshotBase64 || !previewResult?.marker) {
+                return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                    text: "preview_click failed: could not annotate the current screenshot.",
+                }));
+            }
+            const previewEventId = randomUUID();
+            await this.recordPreviewScreenshot(previewResult.screenshotBase64, {
+                runId: context?.runId,
+                sessionId: context?.sessionId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                callId,
+                eventId: previewEventId,
+                captureSource: "preview-click",
+            });
+            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                text: buildPreviewOutputText({
+                    requestedX: previewResult.marker.requestedX,
+                    requestedY: previewResult.marker.requestedY,
+                    renderedX: previewResult.marker.x,
+                    renderedY: previewResult.marker.y,
+                    wasClamped: previewResult.marker.wasClamped,
+                }),
+                annotatedScreenshotBase64: previewResult.screenshotBase64,
+            }));
+        }
+        catch (error) {
+            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                text: `preview_click failed: ${error instanceof Error ? error.message : "unable to capture or annotate the current screen."}`,
+            }));
+        }
+    }
     /**
      * Run a full turn with the CUA model
      * Executes actions until the model stops requesting actions
@@ -156,7 +369,7 @@ export class ExecutionEngine {
                 return newResponseId;
             }
             const items = response.output || [];
-            const computerCalls = extractComputerCalls(items);
+            const actionableItems = extractActionableOutputItems(items);
             // ── Collect pending safety checks ──
             const pendingSafetyChecks = items
                 .filter(item => item.type === "pending_safety_check")
@@ -203,117 +416,18 @@ export class ExecutionEngine {
                     });
                 }
             }
-            if (computerCalls.length === 0) {
+            if (actionableItems.length === 0) {
                 // No actions = turn complete
                 break;
             }
-            // ── Process model actions ──
-            for (const { call_id, actions } of computerCalls) {
-                if (!call_id)
-                    continue;
-                const { actionsToExecute, droppedActions, truncationReason, runtimeNote } = buildStrictModePlan(actions, this.strictMode);
-                let sawExplicitScreenshotAction = false;
-                for (const action of actionsToExecute) {
-                    if (shouldStop()) {
-                        return newResponseId;
-                    }
-                    if (action.type === "screenshot") {
-                        sawExplicitScreenshotAction = true;
-                        addOutput({
-                            type: 'info',
-                            text: 'Capturing screen',
-                            eventType: 'screenshot_captured',
-                            actionType: 'screenshot',
-                            ...eventMeta({
-                                payload: {
-                                    callId: call_id,
-                                    source: 'explicit_action'
-                                }
-                            })
-                        });
-                    }
-                    else {
-                        await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
-                            ...context,
-                            shouldStop,
-                            stepId: stepContext?.stepId,
-                            instructionIndex: stepContext?.instructionIndex
-                        });
-                        // Track action and check for interruption
-                        if (trackAction) {
-                            trackAction(action);
-                        }
-                        if (shouldStop()) {
-                            // User interrupted - stop execution immediately
-                            return newResponseId;
-                        }
-                        // Add delay after UI-changing actions to let the interface update
-                        // before taking the screenshot (except for explicit wait actions which have their own delay)
-                        if (action.type !== "wait") {
-                            await new Promise(resolve => setTimeout(resolve, this.stepDelayMs));
-                        }
-                    }
-                }
-                if (shouldStop()) {
-                    return newResponseId;
-                }
-                const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-                emitDesktopDebug("device.screenshot", "device", {
-                    runId: context?.runId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex
-                }, {
-                    source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
-                    callId: call_id,
-                    width: this.session.deviceInfo?.scaled_width,
-                    height: this.session.deviceInfo?.scaled_height,
-                    base64Length: screenshotBase64.length
-                });
-                await this.recordScreenshot(screenshotBase64, {
-                    runId: context?.runId,
-                    sessionId: context?.sessionId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex,
-                    callId: call_id,
-                    captureSource: sawExplicitScreenshotAction ? "call-output-explicit-action" : "call-output-post-action"
-                });
-                if (runtimeNote) {
-                    const executedActionTypes = actionsToExecute
-                        .map((action) => action?.type)
-                        .filter(Boolean);
-                    const droppedActionTypes = droppedActions
-                        .map((action) => action?.type)
-                        .filter(Boolean);
-                    emitDesktopDebug("cua.strict_mode.truncation", scope, ids, {
-                        callId: call_id,
-                        executedCount: actionsToExecute.length,
-                        droppedCount: droppedActions.length,
-                        executedActionTypes,
-                        droppedActionTypes,
-                        reason: truncationReason,
-                        note: runtimeNote
-                    });
-                }
-                // Build next input: screenshot + any carryover reasoning
-                const selectedCuaModel = process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
-                const input = [{
-                        type: "computer_call_output",
-                        call_id,
-                        output: {
-                            type: "computer_screenshot",
-                            image_url: `data:image/png;base64,${screenshotBase64}`,
-                        },
-                        ...(selectedCuaModel === "computer-use-preview"
-                            ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
-                            : {}),
-                        ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
-                    }];
-                if (runtimeNote) {
-                    input.push({
-                        role: "user",
-                        content: runtimeNote
-                    });
-                }
+            const actionableItem = actionableItems[0];
+            if (!actionableItem?.call_id) {
+                break;
+            }
+            if (actionableItem.type === "preview_click") {
+                const input = [
+                    await this.handlePreviewClick(actionableItem.call_id, actionableItem.arguments, context, stepContext),
+                ];
                 if (shouldStop()) {
                     return newResponseId;
                 }
@@ -330,9 +444,145 @@ export class ExecutionEngine {
                     }
                 });
                 newResponseId = response.id;
-                // Each tool output advances the response chain; process newly returned calls next.
-                break;
+                continue;
+            }
+            const { call_id, actions } = actionableItem;
+            const { actionsToExecute, droppedActions, truncationReason, runtimeNote } = buildStrictModePlan(actions, this.strictMode);
+            let sawExplicitScreenshotAction = false;
+            for (const action of actionsToExecute) {
+                const actionEventId = randomUUID();
+                if (shouldStop()) {
+                    return newResponseId;
+                }
+                if (action.type === "screenshot") {
+                    sawExplicitScreenshotAction = true;
+                }
+                else {
+                    const annotatedScreenshotResult = await this.annotateLatestScreenshot(action, {
+                        runId: context?.runId,
+                        sessionId: context?.sessionId,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex,
+                        eventId: actionEventId,
+                        captureSource: "action-click-annotation"
+                    });
+                    const annotatedScreenshotPath = screenshotPathFromRecorderResult(annotatedScreenshotResult);
+                    const annotatedScreenshotDataUrl = screenshotDataUrlFromRecorderResult(annotatedScreenshotResult);
+                    await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
+                        ...context,
+                        shouldStop,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex,
+                        eventId: actionEventId,
+                        ...(annotatedScreenshotPath ? { screenshotPath: annotatedScreenshotPath } : {}),
+                        ...(context?.includeScreenshotDataUrl && annotatedScreenshotDataUrl ? { screenshotDataUrl: annotatedScreenshotDataUrl } : {})
+                    });
+                    if (trackAction) {
+                        trackAction(action);
+                    }
+                    if (shouldStop()) {
+                        return newResponseId;
+                    }
+                    if (action.type !== "wait") {
+                        await new Promise(resolve => setTimeout(resolve, this.stepDelayMs));
+                    }
+                }
             }
+            if (shouldStop()) {
+                return newResponseId;
+            }
+            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+            emitDesktopDebug("device.screenshot", "device", {
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex
+            }, {
+                source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+                callId: call_id,
+                width: this.session.deviceInfo?.scaled_width,
+                height: this.session.deviceInfo?.scaled_height,
+                base64Length: screenshotBase64.length
+            });
+            const screenshotEventId = randomUUID();
+            const captureSource = sawExplicitScreenshotAction ? "call-output-explicit-action" : "call-output-post-action";
+            const screenshotResult = await this.recordScreenshot(screenshotBase64, {
+                runId: context?.runId,
+                sessionId: context?.sessionId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                eventId: screenshotEventId,
+                captureSource
+            });
+            const screenshotPath = screenshotPathFromRecorderResult(screenshotResult);
+            addOutput({
+                type: 'info',
+                text: 'Capturing screen',
+                eventType: 'screenshot_captured',
+                actionType: 'screenshot',
+                ...eventMeta({
+                    payload: {
+                        eventId: screenshotEventId,
+                        captureSource,
+                        source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+                        ...(typeof screenshotPath === "string" && screenshotPath ? { screenshotPath } : {}),
+                        ...(context?.includeScreenshotDataUrl
+                            ? { screenshotDataUrl: `data:image/png;base64,${screenshotBase64}` }
+                            : {})
+                    }
+                })
+            });
+            if (runtimeNote) {
+                const executedActionTypes = actionsToExecute
+                    .map((action) => action?.type)
+                    .filter(Boolean);
+                const droppedActionTypes = droppedActions
+                    .map((action) => action?.type)
+                    .filter(Boolean);
+                emitDesktopDebug("cua.strict_mode.truncation", scope, ids, {
+                    callId: call_id,
+                    executedCount: actionsToExecute.length,
+                    droppedCount: droppedActions.length,
+                    executedActionTypes,
+                    droppedActionTypes,
+                    reason: truncationReason,
+                    note: runtimeNote
+                });
+            }
+            const selectedCuaModel = process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
+            const input = [{
+                    type: "computer_call_output",
+                    call_id,
+                    output: {
+                        type: "computer_screenshot",
+                        image_url: `data:image/png;base64,${screenshotBase64}`,
+                    },
+                    ...(selectedCuaModel === "computer-use-preview"
+                        ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
+                        : {}),
+                    ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
+                }];
+            if (runtimeNote) {
+                input.push({
+                    role: "user",
+                    content: runtimeNote
+                });
+            }
+            if (shouldStop()) {
+                return newResponseId;
+            }
+            response = await this.sendCUARequest({
+                messages: input,
+                previousResponseId: newResponseId,
+                deviceInfo: this.session.deviceInfo,
+                debugContext: {
+                    scope,
+                    runId: context?.runId,
+                    sessionId: context?.sessionId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }
+            });
+            newResponseId = response.id;
         }
         // ── At end, if last output was only reasoning ──
         const finalItems = response.output || [];

package/build/src/device/android/actions.js

@@ -40,6 +40,9 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform: 'android',
             ...payload
         }
@@ -71,7 +74,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             case "double_click":
                 const realX = Math.round(x / scale);
                 const realY = Math.round(y / scale);
-                addOutput({ type: 'action', text: `Clicking at (${realX}, ${realY})`, ...meta({ x: realX, y: realY, unit: 'px' }) });
+                addOutput({ type: 'action', text: 'Click', ...meta({ x: realX, y: realY, unit: 'px' }) });
                 await adbShell(deviceId, `input tap ${realX} ${realY}`);
                 break;
             case "scroll":
@@ -154,6 +157,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform: 'android'
             }

package/build/src/device/assertions.js

@@ -35,7 +35,7 @@ export function extractFailureDetails(transcript) {
     const parts = recentTranscript.split("ASSERTION RESULT: FAIL");
     return parts[1]?.trim() || "Could not confidently validate the assertion.";
 }
-export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context, stepContext = null) {
+export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMode, context, stepContext = null, eventPayload = {}) {
     const details = extractFailureDetails(transcript);
     const addOutput = context?.addOutput || printCliOutput;
     const meta = {
@@ -46,7 +46,8 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
         payload: {
             assertion: assertionPrompt,
             passed: false,
-            details
+            details,
+            ...eventPayload
         }
     };
     addOutput({ type: 'error', text: '❌ ASSERTION FAILED', ...meta });
@@ -61,7 +62,7 @@ export function handleAssertionFailure(assertionPrompt, transcript, isHeadlessMo
     }
     // Interactive mode: caller should clear remaining instructions
 }
-export function handleAssertionSuccess(assertionPrompt, context = null, stepContext = null) {
+export function handleAssertionSuccess(assertionPrompt, context = null, stepContext = null, eventPayload = {}) {
     const addOutput = context?.addOutput || printCliOutput;
     addOutput({
         type: 'success',
@@ -72,7 +73,8 @@ export function handleAssertionSuccess(assertionPrompt, context = null, stepCont
         instructionIndex: stepContext?.instructionIndex,
         payload: {
             assertion: assertionPrompt,
-            passed: true
+            passed: true,
+            ...eventPayload
         }
     });
 }

package/build/src/device/cloud/actions.js

@@ -54,6 +54,9 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform,
             source: session?.providerId,
             ...payload
@@ -84,7 +87,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             case "double_click": {
                 const targetX = Math.round((action.x / scale) / dpr);
                 const targetY = Math.round((action.y / scale) / dpr);
-                addOutput({ type: "action", text: `Tapping at (${targetX}, ${targetY})`, ...meta({ x: targetX, y: targetY }) });
+                addOutput({ type: "action", text: "Click", ...meta({ x: targetX, y: targetY }) });
                 await session.client.tap(session.sessionId, targetX, targetY);
                 break;
             }
@@ -181,6 +184,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform,
                 source: session.providerId

package/build/src/device/ios/actions.js

@@ -51,6 +51,9 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform: "ios",
             ...payload
         }
@@ -82,7 +85,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                 const pixelY = Math.round(action.y / scale);
                 const pointX = Math.round(pixelX / dpr);
                 const pointY = Math.round(pixelY / dpr);
-                addOutput({ type: "action", text: `Tapping at (${pointX}, ${pointY}) points`, ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
+                addOutput({ type: "action", text: "Click", ...meta({ x: pointX, y: pointY, unit: "points", deviceScale: dpr }) });
                 await appium.tap(session.sessionId, pointX, pointY);
                 break;
             }
@@ -180,6 +183,7 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform: "ios"
             }

package/build/src/device/openai.js

@@ -8,47 +8,87 @@ const cuaDebugTracer = new CuaDebugTracer(logger);
 function getSelectedCuaModel() {
     return process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
 }
-function normalizePreviewMessages(messages, screenshotBase64, includeInitialScreenshot) {
-    const normalized = messages.map((message) => {
-        if (!message?.role) {
-            return message;
-        }
-        const contentItems = Array.isArray(message.content)
-            ? message.content.map((item) => {
-                if (typeof item === "string") {
-                    return { type: "input_text", text: item };
-                }
-                return item;
-            })
-            : [{ type: "input_text", text: String(message.content ?? "") }];
-        return {
-            role: message.role,
-            content: contentItems,
-        };
-    });
-    if (!includeInitialScreenshot || !screenshotBase64) {
-        return normalized;
+function normalizeMessageContent(content) {
+    if (Array.isArray(content)) {
+        return content.map((item) => {
+            if (typeof item === "string") {
+                return { type: "input_text", text: item };
+            }
+            return item;
+        });
     }
-    for (let index = normalized.length - 1; index >= 0; index -= 1) {
-        const item = normalized[index];
+    return [{ type: "input_text", text: String(content ?? "") }];
+}
+function buildScreenshotDataUrl(screenshotBase64) {
+    return `data:image/png;base64,${screenshotBase64}`;
+}
+const previewClickTool = {
+    type: "function",
+    name: "preview_click",
+    description: "Preview where a proposed click would land on the current device screen. Returns an annotated screenshot with a marker at the given coordinates; the device state is unchanged and the click is not executed. Use this before issuing a click action to verify your aim.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            x: { type: "number" },
+            y: { type: "number" },
+        },
+        required: ["x", "y"],
+        additionalProperties: false,
+    },
+};
+function appendInitialScreenshot(input, screenshotBase64) {
+    // Most callers end with the active user message, but some recovery flows can
+    // reset the conversation to system-only state before the next screenshot send.
+    // Attach to the last user message when present, otherwise synthesize one below.
+    for (let index = input.length - 1; index >= 0; index -= 1) {
+        const item = input[index];
         if (item?.role !== "user" || !Array.isArray(item.content))
             continue;
         item.content.push({
             type: "input_image",
-            image_url: `data:image/png;base64,${screenshotBase64}`,
+            image_url: buildScreenshotDataUrl(screenshotBase64),
         });
-        return normalized;
+        return;
     }
-    normalized.push({
+    input.push({
         role: "user",
         content: [{
                 type: "input_image",
-                image_url: `data:image/png;base64,${screenshotBase64}`,
+                image_url: buildScreenshotDataUrl(screenshotBase64),
             }],
     });
-    return normalized;
 }
-function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
+export function buildCuaInput({ messages, screenshotBase64, previousResponseId, callId }) {
+    const input = messages.map((message) => {
+        if (!message?.role) {
+            return message;
+        }
+        return {
+            role: message.role,
+            content: normalizeMessageContent(message.content),
+        };
+    });
+    if (!screenshotBase64) {
+        return input;
+    }
+    if (callId) {
+        input.push({
+            type: "computer_call_output",
+            call_id: callId,
+            output: {
+                type: "computer_screenshot",
+                image_url: buildScreenshotDataUrl(screenshotBase64),
+            },
+        });
+        return input;
+    }
+    if (!previousResponseId) {
+        appendInitialScreenshot(input, screenshotBase64);
+    }
+    return input;
+}
+export function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input }) {
     if (cuaModel === "computer-use-preview") {
         return {
             model: cuaModel,
@@ -56,24 +96,31 @@ function buildCuaRequestParams({ cuaModel, previousResponseId, deviceInfo, input
             input,
             store: true,
             truncation: "auto",
-            tools: [{
+            tools: [
+                {
                     type: "computer_use_preview",
                     display_width: deviceInfo.scaled_width,
                     display_height: deviceInfo.scaled_height,
                     environment: "browser",
-                }],
+                },
+                previewClickTool,
+            ],
             reasoning: { summary: "concise" },
         };
     }
     return {
         model: cuaModel,
+        reasoning: { effort: "medium" },
         previous_response_id: previousResponseId || undefined,
         input,
         store: true,
         truncation: "auto",
-        tools: [{
+        tools: [
+            {
                 type: "computer",
-            }],
+            },
+            previewClickTool,
+        ],
     };
 }
 function shouldMapPreviewAccessError(err) {
@@ -153,20 +200,7 @@ export async function compactAppContext({ contextDocument, taskDescription, toke
 }
 export async function sendCUARequest({ messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
     const cuaModel = getSelectedCuaModel();
-    const includeInitialScreenshot = cuaModel === "computer-use-preview" && !previousResponseId && !callId;
-    const input = cuaModel === "computer-use-preview"
-        ? normalizePreviewMessages(messages, screenshotBase64, includeInitialScreenshot)
-        : [...messages];
-    if (callId && screenshotBase64) {
-        input.push({
-            type: "computer_call_output",
-            call_id: callId,
-            output: {
-                type: "computer_screenshot",
-                image_url: `data:image/png;base64,${screenshotBase64}`,
-            },
-        });
-    }
+    const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
     const requestParams = buildCuaRequestParams({
         cuaModel,
         previousResponseId,

package/build/src/modes/execution-mode.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
 import { sendCUARequest } from "../device/openai.js";
 import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
@@ -14,6 +15,22 @@ export function buildExecutionRequestPayload({ instruction, isAssertionStep, mes
         previousResponseIdToSend: previousResponseId,
     };
 }
+function screenshotPathFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
+            ? result.screenshotPath
+            : null;
+    }
+    return null;
+}
+function screenshotDataUrlFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
+            ? result.screenshotDataUrl
+            : null;
+    }
+    return null;
+}
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -197,6 +214,7 @@ export class ExecutionMode {
         }
         // ── Check for assertion ──
         const isAssertionStep = isAssertion(instruction);
+        const assertionEventId = isAssertionStep ? randomUUID() : null;
         let assertionPrompt = null;
         const transcriptStartIndex = this.session.transcript.length;
         if (isAssertionStep) {
@@ -213,12 +231,27 @@ export class ExecutionMode {
         }
         try {
             const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-            await this.engine.recordScreenshot?.(screenshotBase64, {
+            const screenshotResult = await this.engine.recordScreenshot?.(screenshotBase64, {
                 runId: context?.runId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
+                ...(assertionEventId ? { eventId: assertionEventId } : {}),
                 captureSource: isAssertionStep ? "instruction-input-assertion" : "instruction-input"
             });
+            const assertionScreenshotPath = isAssertionStep ? screenshotPathFromRecorderResult(screenshotResult) : null;
+            const assertionScreenshotDataUrl = isAssertionStep
+                ? screenshotDataUrlFromRecorderResult(screenshotResult) ||
+                    (context?.includeScreenshotDataUrl && assertionScreenshotPath
+                        ? `data:image/png;base64,${screenshotBase64}`
+                        : null)
+                : null;
+            const assertionEventPayload = isAssertionStep
+                ? {
+                    eventId: assertionEventId,
+                    ...(assertionScreenshotPath ? { screenshotPath: assertionScreenshotPath } : {}),
+                    ...(assertionScreenshotDataUrl ? { screenshotDataUrl: assertionScreenshotDataUrl } : {})
+                }
+                : {};
             // Assertions rely on the prior response chain for earlier execution context.
             const { messagesToSend, previousResponseIdToSend } = buildExecutionRequestPayload({
                 instruction,
@@ -253,7 +286,7 @@ export class ExecutionMode {
                 const result = checkAssertionResult(assertionTranscript);
                 if (result.failed) {
                     handleAssertionFailure(assertionPrompt, assertionTranscript, false, // Never exit process - we'll always prompt the user in interactive mode
-                    context, stepContext);
+                    context, stepContext, assertionEventPayload);
                     // In headless mode, exit immediately on assertion failure
                     if (this.isHeadlessMode) {
                         this.stats.assertionsFailed++;
@@ -297,7 +330,7 @@ export class ExecutionMode {
                 }
                 else if (result.passed) {
                     this.stats.assertionsPassed++;
-                    handleAssertionSuccess(assertionPrompt, context, stepContext);
+                    handleAssertionSuccess(assertionPrompt, context, stepContext, assertionEventPayload);
                 }
             }
             // Clear messages after each turn but KEEP the response chain for context

package/build/src/prompts/base.js

@@ -62,6 +62,10 @@ function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
 
   When aiming for visual targets:
   - Reason carefully about the approximate pixel position.
+  - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
+  - Review the returned annotated screenshot before clicking.
+  - Only click if the marker is clearly inside the intended target and not on its edge, border, text, or nearby empty space.
+  - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
   - Click precisely based on your visual estimate.
 
   Available actions: click, scroll, type, keypress, wait, screenshot.

package/build/src/utils/cua-debug-tracer.js

@@ -63,6 +63,19 @@ function summarizeRequestInput(input) {
                 hasSafetyChecks: Array.isArray(item.acknowledged_safety_checks) && item.acknowledged_safety_checks.length > 0
             };
         }
+        if (item?.type === "function_call_output") {
+            const content = Array.isArray(item.output) ? item.output : [];
+            const imageBytes = content
+                .filter((entry) => entry?.type === "input_image" && typeof entry.image_url === "string")
+                .map((entry) => entry.image_url.replace("data:image/png;base64,", "").length);
+            return {
+                type: "function_call_output",
+                callId: item.call_id,
+                contentTypes: content.map((entry) => entry?.type).filter(Boolean),
+                imageBytes,
+                text: summarizeContent(content),
+            };
+        }
         return {
             type: item?.type || "unknown"
         };
@@ -85,6 +98,21 @@ function redactRequestForFileLog(requestParams) {
                     acknowledged_safety_checks: item.acknowledged_safety_checks
                 };
             }
+            if (item.type === "function_call_output" && Array.isArray(item.output)) {
+                return {
+                    ...item,
+                    output: item.output.map((entry) => {
+                        if (entry?.type === "input_image" && typeof entry.image_url === "string") {
+                            const base64Data = entry.image_url.replace("data:image/png;base64,", "");
+                            return {
+                                ...entry,
+                                image_url: `data:image/png;base64,[${base64Data.length} chars]`
+                            };
+                        }
+                        return entry;
+                    })
+                };
+            }
             return item;
         })
     };
@@ -117,6 +145,15 @@ function summarizeResponseOutput(response) {
                 actionType: item.action?.type
             };
         }
+        if (item.type === "function_call") {
+            return {
+                type: "function_call",
+                id: item.id,
+                callId: item.call_id,
+                name: item.name,
+                arguments: truncateForDebug(item.arguments, 300)
+            };
+        }
         if (item.type === "pending_safety_check") {
             return { type: "pending_safety_check", id: item.id, code: item.code };
         }
@@ -175,11 +212,11 @@ export class CuaDebugTracer {
             }));
         });
         const inputCallIds = input
-            .filter((item) => item?.type === "computer_call_output")
+            .filter((item) => item?.type === "computer_call_output" || item?.type === "function_call_output")
             .map((item) => item.call_id)
             .filter(Boolean);
         const inputItemTypes = input.map((item) => item?.type || (item?.role ? `message:${item.role}` : "unknown"));
-        const lastResponseCallIds = Array.isArray(lastResponseMeta?.computerCallIds) ? lastResponseMeta.computerCallIds : [];
+        const lastResponseCallIds = Array.isArray(lastResponseMeta?.toolCallIds) ? lastResponseMeta.toolCallIds : [];
         const missingCallIds = inputCallIds.filter((callId) => !lastResponseCallIds.includes(callId));
         const previousResponseIdMatchesLastResponseId = !previousResponseId || !lastResponseMeta?.id ? null : previousResponseId === lastResponseMeta.id;
         const allInputCallIdsFoundInLastResponse = inputCallIds.length === 0 ? true : missingCallIds.length === 0;
@@ -252,10 +289,12 @@ export class CuaDebugTracer {
     onResponse(trace, response) {
         const outputTypes = (response.output || []).map((item) => item.type);
         const toolCalls = (response.output || [])
-            .filter((item) => item.type === "computer_call")
+            .filter((item) => item.type === "computer_call" || item.type === "function_call")
             .map((item) => ({
             call_id: item.call_id,
-            action_type: item.action?.type
+            action_type: item.action?.type,
+            name: item.name,
+            type: item.type
         }));
         const safetyChecks = (response.output || [])
             .filter((item) => item.type === "pending_safety_check")
@@ -268,7 +307,11 @@ export class CuaDebugTracer {
         this.lastResponseMetaByChain.set(trace.chainKey, {
             id: response.id,
             outputTypes,
-            computerCallIds: toolCalls.map((item) => item.call_id).filter(Boolean),
+            toolCallIds: toolCalls.map((item) => item.call_id).filter(Boolean),
+            computerCallIds: toolCalls
+                .filter((item) => item.type === "computer_call")
+                .map((item) => item.call_id)
+                .filter(Boolean),
             pendingSafetyCheckIds: safetyChecks.map((item) => item.id).filter(Boolean)
         });
         const accountedItems = toolCalls.length + safetyChecks.length;

package/build/src/utils/run-screenshot-recorder.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { mkdir, writeFile } from "node:fs/promises";
+import sharp from "sharp";
 function pad2(value) {
     return String(value).padStart(2, "0");
 }
@@ -32,6 +33,12 @@ function buildCallSegment(callId) {
     }
     return `call-${sanitizeArtifactSegment(callId, "call").slice(0, 24)}`;
 }
+function buildEventSegment(eventId) {
+    if (typeof eventId !== "string" || !eventId.trim()) {
+        return null;
+    }
+    return `event-${sanitizeArtifactSegment(eventId, "event").slice(0, 36)}`;
+}
 function buildScreenshotFileName(sequence, metadata = {}) {
     const segments = [
         String(sequence).padStart(4, "0"),
@@ -50,8 +57,119 @@ function buildScreenshotFileName(sequence, metadata = {}) {
     if (callSegment) {
         segments.push(callSegment);
     }
+    const eventSegment = buildEventSegment(metadata.eventId);
+    if (eventSegment) {
+        segments.push(eventSegment);
+    }
     return `${segments.join("_")}.png`;
 }
+function clampCoordinate(value, max) {
+    if (!Number.isFinite(value) || max <= 0) {
+        return 0;
+    }
+    return Math.max(0, Math.min(max - 1, Math.round(value)));
+}
+function buildMarkerPlacement(marker = {}, width, height) {
+    const requestedX = Number(marker.x);
+    const requestedY = Number(marker.y);
+    const x = clampCoordinate(requestedX, width);
+    const y = clampCoordinate(requestedY, height);
+    return {
+        requestedX,
+        requestedY,
+        x,
+        y,
+        wasClamped: !Number.isFinite(requestedX) || !Number.isFinite(requestedY) || x !== Math.round(requestedX) || y !== Math.round(requestedY),
+    };
+}
+const MARKER_APPEARANCES = {
+    click: {
+        haloFill: "#39ff14",
+        haloOpacity: 0.28,
+        ringStroke: "#39ff14",
+        centerDotFill: "#07131f",
+    },
+    preview: {
+        haloFill: "#00d8ff",
+        haloOpacity: 0.28,
+        ringStroke: "#00d8ff",
+        centerDotFill: "#07131f",
+    },
+};
+function getMarkerAppearance(marker = {}) {
+    if (marker?.style === "preview") {
+        return MARKER_APPEARANCES.preview;
+    }
+    return MARKER_APPEARANCES.click;
+}
+function buildClickMarkerSvg({ width, height, x, y, radius, marker }) {
+    const haloRadius = radius + 8;
+    const whiteStrokeWidth = Math.max(5, Math.round(radius * 0.34));
+    const ringStrokeWidth = Math.max(3, Math.round(radius * 0.18));
+    const centerDotRadius = Math.max(2, Math.round(radius * 0.22));
+    const centerDotStrokeWidth = Math.max(2, Math.round(radius * 0.1));
+    const appearance = getMarkerAppearance(marker);
+    return `
+    <svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}">
+      <circle cx="${x}" cy="${y}" r="${haloRadius}" fill="${appearance.haloFill}" opacity="${appearance.haloOpacity}" />
+      <circle cx="${x}" cy="${y}" r="${radius}" fill="none" stroke="#ffffff" stroke-width="${whiteStrokeWidth}" />
+      <circle cx="${x}" cy="${y}" r="${radius}" fill="none" stroke="${appearance.ringStroke}" stroke-width="${ringStrokeWidth}" />
+      <circle cx="${x}" cy="${y}" r="${centerDotRadius}" fill="${appearance.centerDotFill}" stroke="#ffffff" stroke-width="${centerDotStrokeWidth}" />
+    </svg>
+  `.trim();
+}
+async function renderAnnotatedScreenshot(image, marker = {}) {
+    const metadata = await image.metadata();
+    const width = Number(metadata.width) || 0;
+    const height = Number(metadata.height) || 0;
+    if (width <= 0 || height <= 0) {
+        return null;
+    }
+    const placement = buildMarkerPlacement(marker, width, height);
+    const radius = Math.max(14, Math.round(Math.min(width, height) * 0.055));
+    const overlay = Buffer.from(buildClickMarkerSvg({ width, height, x: placement.x, y: placement.y, radius, marker }));
+    const outputBuffer = await image
+        .composite([{ input: overlay }])
+        .png()
+        .toBuffer();
+    return {
+        outputBuffer,
+        width,
+        height,
+        marker: placement,
+    };
+}
+export async function createAnnotatedScreenshotBuffer(filePath, marker = {}) {
+    if (typeof filePath !== "string" || !filePath.trim()) {
+        return null;
+    }
+    const result = await renderAnnotatedScreenshot(sharp(filePath), marker);
+    if (!result) {
+        return null;
+    }
+    return result.outputBuffer;
+}
+export async function createAnnotatedScreenshotFromBase64(screenshotBase64, marker = {}) {
+    if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+        return null;
+    }
+    const result = await renderAnnotatedScreenshot(sharp(Buffer.from(screenshotBase64, "base64")), marker);
+    if (!result) {
+        return null;
+    }
+    return {
+        ...result,
+        screenshotBase64: result.outputBuffer.toString("base64"),
+    };
+}
+export async function annotateScreenshotArtifact(filePath, marker = {}) {
+    const outputBuffer = await createAnnotatedScreenshotBuffer(filePath, marker);
+    if (!outputBuffer) {
+        return null;
+    }
+    await writeFile(filePath, outputBuffer);
+    return filePath;
+}
 export function createDebugScreenshotRecorder({ directoryPath }) {
     let nextSequence = 1;
     let ensuredDirectoryPromise = null;
@@ -76,6 +194,23 @@ export function createDebugScreenshotRecorder({ directoryPath }) {
             const filePath = path.join(directoryPath, fileName);
             await writeFile(filePath, Buffer.from(screenshotBase64, "base64"));
             return filePath;
+        },
+        async saveAnnotatedScreenshot(sourceFilePath, marker = {}, metadata = {}) {
+            if (typeof sourceFilePath !== "string" || !sourceFilePath.trim()) {
+                return null;
+            }
+            await ensureDirectory();
+            const outputBuffer = await createAnnotatedScreenshotBuffer(sourceFilePath, marker);
+            if (!outputBuffer) {
+                return null;
+            }
+            const fileName = buildScreenshotFileName(nextSequence++, metadata);
+            const filePath = path.join(directoryPath, fileName);
+            await writeFile(filePath, outputBuffer);
+            return {
+                artifactPaths: [filePath],
+                annotationSourcePaths: [sourceFilePath],
+            };
         }
     };
 }
@@ -93,6 +228,32 @@ export function createCompositeScreenshotRecorder({ recorders }) {
             }
             const results = await Promise.all(activeRecorders.map((recorder) => recorder.saveScreenshot(screenshotBase64, metadata)));
             return results.filter(Boolean);
+        },
+        async saveAnnotatedScreenshot(sourceFilePath, marker = {}, metadata = {}) {
+            if (typeof sourceFilePath !== "string" || !sourceFilePath.trim()) {
+                return null;
+            }
+            const results = await Promise.all(activeRecorders.map((recorder) => recorder.saveAnnotatedScreenshot?.(sourceFilePath, marker, metadata)));
+            const filteredResults = results.filter(Boolean);
+            if (filteredResults.length === 0) {
+                return null;
+            }
+            return {
+                artifactPaths: filteredResults.flatMap((result) => {
+                    if (result && typeof result === "object" && !Array.isArray(result)) {
+                        return Array.isArray(result.artifactPaths) ? result.artifactPaths.filter(Boolean) : [];
+                    }
+                    return typeof result === "string" ? [result] : [];
+                }),
+                annotationSourcePaths: filteredResults.flatMap((result) => {
+                    if (result && typeof result === "object" && !Array.isArray(result) && Array.isArray(result.annotationSourcePaths)) {
+                        return result.annotationSourcePaths.filter(Boolean);
+                    }
+                    return [];
+                }),
+                screenshotPath: filteredResults.find((result) => result?.screenshotPath)?.screenshotPath ?? null,
+                screenshotDataUrl: filteredResults.find((result) => result?.screenshotDataUrl)?.screenshotDataUrl ?? null,
+            };
         }
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.5.0",
+  "version": "2.6.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",