@loadmill/droid-cua 2.9.0 → 2.10.1-rc.8

package/build/src/core/execution-engine.js

@@ -1,21 +1,28 @@
 import { randomUUID } from "node:crypto";
-import path from "path";
-import { writeFile } from "fs/promises";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
+import { getBrowserInfo, navigateToUrl as navigateBrowserToUrl } from "../browser/connection.js";
 import { handleModelAction } from "../device/actions.js";
 import { sendCUARequest } from "../device/openai.js";
+import { executeLoadmillInstruction } from "../device/loadmill.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { printCliOutput } from "../utils/console-output.js";
-import { annotateScreenshotArtifact, createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
 import { CAPTURE_SOURCES } from "./capture-sources.js";
+import { ScreenshotStore } from "./screenshot-store.js";
+import { handlePreviewClick } from "./tool-handlers/preview-click.js";
+import { handleNavigateToolCall } from "./tool-handlers/navigate.js";
+import { handleAssertionToolCall } from "./tool-handlers/assert.js";
+import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
+import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
+import { buildDebugContext, buildTextToolOutputItem, } from "./tool-handlers/utils.js";
 function extractActionableOutputItems(items) {
     const entries = [];
     for (const item of items) {
-        if (item?.type === "function_call" && item.name === "preview_click") {
+        if (item?.type === "function_call") {
             entries.push({
-                type: "preview_click",
+                type: "function_call",
                 call_id: item.call_id || item.id,
+                name: item.name,
                 arguments: item.arguments,
             });
             continue;
@@ -42,144 +49,10 @@ function extractActionableOutputItems(items) {
     }
     return entries;
 }
-function buildPreviewClickValidationError(argumentsText) {
-    if (!argumentsText || typeof argumentsText !== "string") {
-        return "Invalid preview_click arguments: expected a JSON object with numeric x and y.";
-    }
-    return `Invalid preview_click arguments: ${argumentsText}`;
-}
-function parsePreviewClickArguments(argumentsText) {
-    let parsedArguments;
-    try {
-        parsedArguments = JSON.parse(argumentsText);
-    }
-    catch {
-        return {
-            ok: false,
-            message: buildPreviewClickValidationError("arguments could not be parsed as JSON; expected numeric x and y."),
-        };
-    }
-    const x = parsedArguments?.x;
-    const y = parsedArguments?.y;
-    if (!Number.isFinite(x) || !Number.isFinite(y)) {
-        return {
-            ok: false,
-            message: buildPreviewClickValidationError("x and y must both be numeric."),
-        };
-    }
-    return {
-        ok: true,
-        x,
-        y,
-    };
-}
-function buildFunctionCallOutputItem(callId, output) {
-    return {
-        type: "function_call_output",
-        call_id: callId,
-        output,
-    };
-}
-function buildPreviewOutputText({ requestedX, requestedY, renderedX, renderedY, wasClamped }) {
-    const baseText = `Marker placed at (${renderedX}, ${renderedY}) on the current screen. The click has not been executed.`;
-    if (!wasClamped) {
-        return baseText;
-    }
-    return `${baseText} Requested point (${requestedX}, ${requestedY}) was off-screen; marker rendered at (${renderedX}, ${renderedY}).`;
-}
-function buildPreviewOutputContent({ text, annotatedScreenshotBase64 = null }) {
-    const content = [{
-            type: "input_text",
-            text,
-        }];
-    if (annotatedScreenshotBase64) {
-        content.push({
-            type: "input_image",
-            image_url: `data:image/png;base64,${annotatedScreenshotBase64}`,
-            detail: "high",
-        });
-    }
-    return content;
-}
-function getScopeAndIds(context = null, stepContext = null) {
-    const scope = context?.sessionId ? "design" : "execution";
-    const ids = scope === "design"
-        ? {
-            sessionId: context?.sessionId,
-            stepId: stepContext?.stepId,
-            instructionIndex: stepContext?.instructionIndex
-        }
-        : {
-            runId: context?.runId,
-            stepId: stepContext?.stepId,
-            instructionIndex: stepContext?.instructionIndex
-        };
-    return { scope, ids };
-}
-function buildStrictModePlan(actions = [], strictMode = false) {
-    if (!strictMode || actions.length === 0) {
-        return {
-            actionsToExecute: actions,
-            droppedActions: [],
-            truncationReason: null,
-            runtimeNote: null
-        };
-    }
-    const actionsToExecute = [actions[0]];
-    const droppedActions = actions.slice(1);
-    if (droppedActions.length === 0) {
-        return {
-            actionsToExecute,
-            droppedActions,
-            truncationReason: null,
-            runtimeNote: null
-        };
-    }
-    const leadingActionType = actionsToExecute[0]?.type;
-    const truncationReason = leadingActionType === "screenshot"
-        ? "leading_screenshot_reobserve"
-        : "post_first_action_reobserve";
-    const droppedActionTypes = droppedActions
-        .map((action) => action?.type)
-        .filter(Boolean);
-    const runtimeNote = `Strict Mode: I executed only the first ${leadingActionType === "screenshot" ? "screenshot request" : "action"} from your previous chain and intentionally skipped the remaining ${droppedActions.length} action${droppedActions.length === 1 ? "" : "s"} (${droppedActionTypes.join(", ")}) so I could re-observe the device before continuing. Base your next step only on what is visible now.`;
-    return {
-        actionsToExecute,
-        droppedActions,
-        truncationReason,
-        runtimeNote
-    };
-}
-function normalizeArtifactPaths(result) {
-    if (result && typeof result === "object" && !Array.isArray(result)) {
-        if (Array.isArray(result.annotationSourcePaths)) {
-            return normalizeArtifactPaths(result.annotationSourcePaths);
-        }
-        return normalizeArtifactPaths(result.artifactPaths);
-    }
-    if (Array.isArray(result)) {
-        return result.filter((value) => typeof value === "string" && value.trim());
-    }
-    if (typeof result === "string" && result.trim()) {
-        return [result];
-    }
-    return [];
-}
-function screenshotPathFromRecorderResult(result) {
-    if (result && typeof result === "object" && !Array.isArray(result)) {
-        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
-            ? result.screenshotPath
-            : null;
-    }
-    return typeof result === "string" && result.trim() ? result : null;
-}
-function screenshotDataUrlFromRecorderResult(result) {
-    if (result && typeof result === "object" && !Array.isArray(result)) {
-        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
-            ? result.screenshotDataUrl
-            : null;
-    }
-    return null;
+function collectPendingSafetyChecks(items = []) {
+    return items
+        .filter((item) => item.type === "pending_safety_check")
+        .map((item) => ({ id: item.id }));
 }
 export class ExecutionEngine {
     constructor(session, options = {}) {
@@ -187,206 +60,239 @@ export class ExecutionEngine {
         this.recordScreenshots = options.recordScreenshots || false;
         this.screenshotDir = options.screenshotDir || null;
         this.screenshotRecorder = options.screenshotRecorder || null;
-        this.strictMode = options.strictMode === true;
+        this.screenshotStore = options.screenshotStore || new ScreenshotStore({
+            recordScreenshots: this.recordScreenshots,
+            screenshotDir: this.screenshotDir,
+            screenshotRecorder: this.screenshotRecorder,
+        });
         this.stepDelayMs = options.stepDelayMs ?? getConfiguredStepDelayMs();
-        this.reportedScreenshotWriteError = false;
-        this.reportedScreenshotAnnotateError = false;
-        this.lastRecordedScreenshotPaths = [];
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
         this.reviseTestScript = options.reviseTestScript || null;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
+        this.getBrowserInfo = options.getBrowserInfo || getBrowserInfo;
+        this.navigateToUrl = options.navigateToUrl || navigateBrowserToUrl;
+        this.executeLoadmillInstruction = options.executeLoadmillInstruction || executeLoadmillInstruction;
+        this.toolCapabilities = options.toolCapabilities || {};
+        this.baseSystemPrompt = options.baseSystemPrompt || session?.systemPrompt || null;
+        this.isHeadlessMode = options.isHeadlessMode === true;
     }
-    async saveScreenshotArtifact(screenshotBase64, metadata = {}, options = {}) {
-        if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+    async recordScreenshot(screenshotBase64, metadata = {}) {
+        return await this.screenshotStore.saveScreenshot(screenshotBase64, metadata);
+    }
+    async annotateScreenshot(currentScreenshot, action, metadata = {}) {
+        if (!(action?.type === "click" || action?.type === "double_click")) {
             return null;
         }
-        const updateLastRecordedScreenshotPaths = options.updateLastRecordedScreenshotPaths !== false;
-        const previousRecordedScreenshotPaths = updateLastRecordedScreenshotPaths
-            ? null
-            : [...this.lastRecordedScreenshotPaths];
-        try {
-            if (this.screenshotRecorder?.saveScreenshot) {
-                const result = await this.screenshotRecorder.saveScreenshot(screenshotBase64, metadata);
-                if (updateLastRecordedScreenshotPaths) {
-                    this.lastRecordedScreenshotPaths = normalizeArtifactPaths(result);
-                }
-                return result;
-            }
-            if (this.recordScreenshots && this.screenshotDir) {
-                const framePath = path.join(this.screenshotDir, `frame_${String(Date.now())}.png`);
-                await writeFile(framePath, Buffer.from(screenshotBase64, "base64"));
-                if (updateLastRecordedScreenshotPaths) {
-                    this.lastRecordedScreenshotPaths = [framePath];
-                }
-                return framePath;
-            }
-        }
-        catch (error) {
-            if (updateLastRecordedScreenshotPaths) {
-                this.lastRecordedScreenshotPaths = [];
-            }
-            else {
-                this.lastRecordedScreenshotPaths = previousRecordedScreenshotPaths;
-            }
-            if (!this.reportedScreenshotWriteError) {
-                this.reportedScreenshotWriteError = true;
-                const scope = typeof metadata.sessionId === "string" ? "design" : "execution";
-                const ids = scope === "design"
-                    ? {
-                        sessionId: metadata.sessionId,
-                        stepId: metadata.stepId,
-                        instructionIndex: metadata.instructionIndex
-                    }
-                    : {
-                        runId: metadata.runId,
-                        stepId: metadata.stepId,
-                        instructionIndex: metadata.instructionIndex
-                    };
-                emitDesktopDebug("run.screenshot_write.error", scope, ids, {
-                    captureSource: metadata.captureSource ?? null,
-                    message: error instanceof Error ? error.message : "Failed to persist screenshot"
-                });
-            }
-        }
-        return null;
+        return await this.screenshotStore.annotateScreenshot(currentScreenshot, { x: action.x, y: action.y, style: "click" }, { ...metadata, actionType: action.type });
     }
-    async recordScreenshot(screenshotBase64, metadata = {}) {
-        return await this.saveScreenshotArtifact(screenshotBase64, metadata);
+    /**
+     * Dispatches a function-call tool to its handler. Each handler returns a
+     * discriminated outcome that runFullTurn interprets below. Continue and
+     * close_turn outcomes may also return currentScreenshot to update the
+     * parent turn's explicit screenshot state.
+     */
+    async handleFunctionToolCall(toolName, callId, argumentsText, context = null, stepContext = null) {
+        const handlerArgs = { engine: this, callId, argumentsText, context, stepContext };
+        switch (toolName) {
+            case "preview_click":
+                return await handlePreviewClick(handlerArgs);
+            case "assert":
+                return await handleAssertionToolCall(handlerArgs);
+            case "navigate":
+                return await handleNavigateToolCall(handlerArgs);
+            case "loadmill":
+                return await handleLoadmillToolCall(handlerArgs);
+            case "test_design_complete":
+                return await handleTestDesignCompleteToolCall(handlerArgs);
+            default:
+                return {
+                    kind: "continue",
+                    outputItem: buildTextToolOutputItem(callId, `Unsupported tool call: ${toolName}`),
+                };
+        }
     }
-    async recordPreviewScreenshot(screenshotBase64, metadata = {}) {
-        return await this.saveScreenshotArtifact(screenshotBase64, metadata, {
-            updateLastRecordedScreenshotPaths: false,
+    async requestNextTurn(messages, previousResponseId, context = null, stepContext = null, options = {}) {
+        return await this.sendCUARequest({
+            messages,
+            previousResponseId,
+            deviceInfo: this.session.deviceInfo,
+            debugContext: options.debugContextOverride ?? buildDebugContext(context, stepContext),
+            toolCapabilities: options.toolCapabilitiesOverride ?? this.toolCapabilities,
+            includeComputerTool: options.includeComputerToolOverride ?? true,
         });
     }
-    async annotateLatestScreenshot(action, metadata = {}) {
-        if (!(action?.type === "click" || action?.type === "double_click") || this.lastRecordedScreenshotPaths.length === 0) {
-            return null;
-        }
-        try {
-            const marker = {
-                x: action.x,
-                y: action.y,
-                style: "click",
-            };
-            const sourcePath = this.lastRecordedScreenshotPaths.find((filePath) => typeof filePath === "string" && filePath.trim());
-            if (sourcePath && this.screenshotRecorder?.saveAnnotatedScreenshot) {
-                return await this.screenshotRecorder.saveAnnotatedScreenshot(sourcePath, marker, metadata);
-            }
-            await Promise.all(this.lastRecordedScreenshotPaths.map((filePath) => annotateScreenshotArtifact(filePath, marker)));
-            return null;
-        }
-        catch (error) {
-            if (!this.reportedScreenshotAnnotateError) {
-                this.reportedScreenshotAnnotateError = true;
-                const scope = typeof metadata.sessionId === "string" ? "design" : "execution";
-                const ids = scope === "design"
-                    ? {
-                        sessionId: metadata.sessionId,
-                        stepId: metadata.stepId,
-                        instructionIndex: metadata.instructionIndex
-                    }
-                    : {
-                        runId: metadata.runId,
-                        stepId: metadata.stepId,
-                        instructionIndex: metadata.instructionIndex
-                    };
-                emitDesktopDebug("run.screenshot_annotate.error", scope, ids, {
-                    actionType: action?.type ?? null,
-                    x: action?.x ?? null,
-                    y: action?.y ?? null,
-                    message: error instanceof Error ? error.message : "Failed to annotate screenshot"
-                });
+    async executeActionBatch(actions, currentScreenshot, shouldStop, context = null, stepContext = null, trackAction = null) {
+        let sawExplicitScreenshotAction = false;
+        for (const action of actions) {
+            const actionEventId = randomUUID();
+            if (shouldStop()) {
+                return { stopped: true, sawExplicitScreenshotAction };
             }
-        }
-        return null;
-    }
-    async handlePreviewClick(callId, argumentsText, context = null, stepContext = null) {
-        const parsedArguments = parsePreviewClickArguments(argumentsText);
-        if (!parsedArguments.ok) {
-            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({ text: parsedArguments.message }));
-        }
-        try {
-            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-            const previewResult = await createAnnotatedScreenshotFromBase64(screenshotBase64, {
-                x: parsedArguments.x,
-                y: parsedArguments.y,
-                style: "preview",
-            });
-            if (!previewResult?.screenshotBase64 || !previewResult?.marker) {
-                return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
-                    text: "preview_click failed: could not annotate the current screenshot.",
-                }));
+            if (action.type === "screenshot") {
+                sawExplicitScreenshotAction = true;
+                continue;
             }
-            const previewEventId = randomUUID();
-            const previewScreenshotResult = await this.recordPreviewScreenshot(previewResult.screenshotBase64, {
+            const annotatedScreenshot = await this.annotateScreenshot(currentScreenshot, action, {
                 runId: context?.runId,
                 sessionId: context?.sessionId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
-                callId,
-                eventId: previewEventId,
-                captureSource: CAPTURE_SOURCES.PREVIEW_CLICK,
+                eventId: actionEventId,
+                captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION
             });
-            const previewText = buildPreviewOutputText({
-                requestedX: previewResult.marker.requestedX,
-                requestedY: previewResult.marker.requestedY,
-                renderedX: previewResult.marker.x,
-                renderedY: previewResult.marker.y,
-                wasClamped: previewResult.marker.wasClamped,
+            await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
+                ...context,
+                shouldStop,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                eventId: actionEventId,
+                captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
+                ...(annotatedScreenshot?.screenshotPath ? { screenshotPath: annotatedScreenshot.screenshotPath } : {}),
+                ...(context?.includeScreenshotDataUrl && annotatedScreenshot?.screenshotDataUrl ? { screenshotDataUrl: annotatedScreenshot.screenshotDataUrl } : {})
             });
-            const addOutput = context?.addOutput || null;
-            if (addOutput) {
-                const previewScreenshotPath = screenshotPathFromRecorderResult(previewScreenshotResult);
-                const previewScreenshotDataUrl = context?.includeScreenshotDataUrl
-                    ? `data:image/png;base64,${previewResult.screenshotBase64}`
-                    : null;
+            if (trackAction) {
+                trackAction(action);
+            }
+            if (shouldStop()) {
+                return { stopped: true, sawExplicitScreenshotAction };
+            }
+            if (action.type !== "wait") {
+                await new Promise((resolve) => setTimeout(resolve, this.stepDelayMs));
+            }
+        }
+        return { stopped: false, sawExplicitScreenshotAction };
+    }
+    async buildActionFollowUp(callId, pendingSafetyChecks, sawExplicitScreenshotAction, addOutput, eventMeta, context = null, stepContext = null) {
+        const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+        emitDesktopDebug("device.screenshot", "device", {
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex
+        }, {
+            source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+            callId,
+            width: this.session.deviceInfo?.scaled_width,
+            height: this.session.deviceInfo?.scaled_height,
+            base64Length: screenshotBase64.length
+        });
+        const screenshotEventId = randomUUID();
+        const captureSource = sawExplicitScreenshotAction ? CAPTURE_SOURCES.CALL_OUTPUT_EXPLICIT_ACTION : CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION;
+        const currentScreenshot = await this.recordScreenshot(screenshotBase64, {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+            eventId: screenshotEventId,
+            captureSource
+        });
+        addOutput({
+            type: "info",
+            text: "Capturing screen",
+            eventType: "screenshot_captured",
+            actionType: "screenshot",
+            ...eventMeta({
+                payload: {
+                    eventId: screenshotEventId,
+                    captureSource,
+                    source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+                    ...(currentScreenshot?.screenshotPath ? { screenshotPath: currentScreenshot.screenshotPath } : {}),
+                    ...(context?.includeScreenshotDataUrl
+                        ? { screenshotDataUrl: `data:image/png;base64,${screenshotBase64}` }
+                        : {})
+                }
+            })
+        });
+        return {
+            input: [{
+                    type: "computer_call_output",
+                    call_id: callId,
+                    output: {
+                        type: "computer_screenshot",
+                        image_url: `data:image/png;base64,${screenshotBase64}`,
+                    },
+                    ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
+                }],
+            currentScreenshot,
+        };
+    }
+    emitResponseOutputItems(items, addOutput, eventMeta, options = {}) {
+        for (const item of items) {
+            if (item.type === "reasoning") {
+                for (const entry of item.summary) {
+                    if (entry.type !== "summary_text")
+                        continue;
+                    addOutput({
+                        type: "reasoning",
+                        text: entry.text,
+                        eventType: "reasoning",
+                        ...eventMeta(),
+                    });
+                    this.session.addToTranscript(`[Reasoning] ${entry.text}`);
+                }
+                continue;
+            }
+            if (item.type === "message") {
+                const textPart = item.content.find((contentItem) => contentItem.type === "output_text");
+                if (!textPart)
+                    continue;
+                if (!options.suppressAssistantOutput) {
+                    addOutput({
+                        type: "assistant",
+                        text: textPart.text,
+                        eventType: "assistant_message",
+                        ...eventMeta(),
+                    });
+                }
+                this.session.addToTranscript(`[Assistant] ${textPart.text}`);
+                continue;
+            }
+            if (item.type === "pending_safety_check") {
                 addOutput({
-                    type: "action",
-                    text: "Click",
-                    eventType: "tool_call",
-                    runId: context?.runId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex,
-                    payload: {
-                        platform: this.session.deviceInfo?.platform,
-                        toolName: "preview_click",
-                        captureSource: CAPTURE_SOURCES.PREVIEW_CLICK,
-                        requestedX: previewResult.marker.requestedX,
-                        requestedY: previewResult.marker.requestedY,
-                        renderedX: previewResult.marker.x,
-                        renderedY: previewResult.marker.y,
-                        wasClamped: previewResult.marker.wasClamped,
-                        ...(previewScreenshotPath ? { screenshotPath: previewScreenshotPath } : {}),
-                        ...(previewScreenshotDataUrl ? { screenshotDataUrl: previewScreenshotDataUrl } : {})
-                    }
+                    type: "warning",
+                    text: `⚠️ Safety check: ${item.code} - ${item.message}`,
+                    eventType: "system_message",
+                    ...eventMeta({
+                        payload: {
+                            id: item.id,
+                            code: item.code,
+                            message: item.message,
+                        },
+                    }),
                 });
             }
-            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
-                text: previewText,
-                annotatedScreenshotBase64: previewResult.screenshotBase64,
-            }));
-        }
-        catch (error) {
-            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
-                text: `preview_click failed: ${error instanceof Error ? error.message : "unable to capture or annotate the current screen."}`,
-            }));
         }
     }
     /**
-     * Run a full turn with the CUA model
-     * Executes actions until the model stops requesting actions
-     * Returns the new response ID for chaining
-     * @param {Object} response - The CUA response
-     * @param {Function} trackAction - Optional callback to track actions for stuck detection
-     * @param {Object} context - Optional Ink context for output
+     * Run a full turn with the CUA model. Executes actions until the model
+     * stops requesting actions, then returns { responseId, summary, currentScreenshot }.
+     *
+     * The summary's shape depends on which (if any) tool handler ran:
+     *   - assertions: summary.assertionStatus = "passed" | "failed" | "failed_skipped" | "retry"
+     *   - failures:   summary.failed = true, summary.error = string
+     *   - retries:    summary.retryInstruction = true (caller should rerun the instruction)
+     *   - navigation: summary.preserveSessionMessages = true (skip the usual message reset)
+     *
+     * @param {Object} response - The CUA response that opens this turn.
+     * @param {Function} trackAction - Optional callback for tracking actions / interrupting.
+     * @param {Object} context - Optional Ink context for output.
+     * @param {Object} options.currentScreenshot - Screenshot object to annotate during this turn.
      */
-    async runFullTurn(response, trackAction = null, context = null, stepContext = null) {
+    async runFullTurn(response, trackAction = null, context = null, stepContext = null, options = {}) {
         const addOutput = context?.addOutput || printCliOutput;
         let newResponseId = response.id;
+        let currentScreenshot = options.currentScreenshot ?? null;
         const shouldStop = () => Boolean(trackAction?.());
-        const { scope, ids } = getScopeAndIds(context, stepContext);
+        const summary = {
+            responseId: newResponseId,
+            invokedFunctionTools: [],
+            preserveSessionMessages: false,
+        };
+        const finalize = () => {
+            summary.responseId = newResponseId;
+            return { responseId: newResponseId, summary, currentScreenshot };
+        };
         const eventMeta = (extra = {}) => ({
             runId: context?.runId,
             stepId: stepContext?.stepId,
@@ -396,56 +302,12 @@ export class ExecutionEngine {
         while (true) {
             // Check for interruption before processing next batch of actions
             if (shouldStop()) {
-                return newResponseId;
+                return finalize();
             }
             const items = response.output || [];
             const actionableItems = extractActionableOutputItems(items);
-            // ── Collect pending safety checks ──
-            const pendingSafetyChecks = items
-                .filter(item => item.type === "pending_safety_check")
-                .map(item => ({ id: item.id }));
-            // ── Print assistant output ──
-            for (const item of items) {
-                if (item.type === "reasoning") {
-                    for (const entry of item.summary) {
-                        if (entry.type === "summary_text") {
-                            addOutput({
-                                type: 'reasoning',
-                                text: entry.text,
-                                eventType: 'reasoning',
-                                ...eventMeta()
-                            });
-                            this.session.addToTranscript(`[Reasoning] ${entry.text}`);
-                        }
-                    }
-                }
-                else if (item.type === "message") {
-                    const textPart = item.content.find(c => c.type === "output_text");
-                    if (textPart) {
-                        addOutput({
-                            type: 'assistant',
-                            text: textPart.text,
-                            eventType: 'assistant_message',
-                            ...eventMeta()
-                        });
-                        this.session.addToTranscript(`[Assistant] ${textPart.text}`);
-                    }
-                }
-                else if (item.type === "pending_safety_check") {
-                    addOutput({
-                        type: 'warning',
-                        text: `⚠️ Safety check: ${item.code} - ${item.message}`,
-                        eventType: 'system_message',
-                        ...eventMeta({
-                            payload: {
-                                id: item.id,
-                                code: item.code,
-                                message: item.message
-                            }
-                        })
-                    });
-                }
-            }
+            const pendingSafetyChecks = collectPendingSafetyChecks(items);
+            this.emitResponseOutputItems(items, addOutput, eventMeta, options);
             if (actionableItems.length === 0) {
                 // No actions = turn complete
                 break;
@@ -454,161 +316,60 @@ export class ExecutionEngine {
             if (!actionableItem?.call_id) {
                 break;
             }
-            if (actionableItem.type === "preview_click") {
-                const input = [
-                    await this.handlePreviewClick(actionableItem.call_id, actionableItem.arguments, context, stepContext),
-                ];
-                if (shouldStop()) {
-                    return newResponseId;
+            if (actionableItem.type === "function_call") {
+                const toolResult = await this.handleFunctionToolCall(actionableItem.name, actionableItem.call_id, actionableItem.arguments, context, stepContext);
+                summary.invokedFunctionTools.push(actionableItem.name);
+                if (toolResult.summaryUpdates) {
+                    Object.assign(summary, toolResult.summaryUpdates);
+                }
+                if ((toolResult.kind === "continue" || toolResult.kind === "close_turn") && toolResult.currentScreenshot) {
+                    currentScreenshot = toolResult.currentScreenshot;
                 }
-                response = await this.sendCUARequest({
-                    messages: input,
-                    previousResponseId: newResponseId,
-                    deviceInfo: this.session.deviceInfo,
-                    debugContext: {
-                        scope,
-                        runId: context?.runId,
-                        sessionId: context?.sessionId,
-                        stepId: stepContext?.stepId,
-                        instructionIndex: stepContext?.instructionIndex
-                    }
-                });
-                newResponseId = response.id;
-                continue;
-            }
-            const { call_id, actions } = actionableItem;
-            const { actionsToExecute, droppedActions, truncationReason, runtimeNote } = buildStrictModePlan(actions, this.strictMode);
-            let sawExplicitScreenshotAction = false;
-            for (const action of actionsToExecute) {
-                const actionEventId = randomUUID();
                 if (shouldStop()) {
-                    return newResponseId;
+                    return finalize();
                 }
-                if (action.type === "screenshot") {
-                    sawExplicitScreenshotAction = true;
+                if (toolResult.kind === "fail") {
+                    summary.failed = true;
+                    summary.error = toolResult.error;
+                    break;
                 }
-                else {
-                    const annotatedScreenshotResult = await this.annotateLatestScreenshot(action, {
-                        runId: context?.runId,
-                        sessionId: context?.sessionId,
-                        stepId: stepContext?.stepId,
-                        instructionIndex: stepContext?.instructionIndex,
-                        eventId: actionEventId,
-                        captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION
-                    });
-                    const annotatedScreenshotPath = screenshotPathFromRecorderResult(annotatedScreenshotResult);
-                    const annotatedScreenshotDataUrl = screenshotDataUrlFromRecorderResult(annotatedScreenshotResult);
-                    await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
-                        ...context,
-                        shouldStop,
-                        stepId: stepContext?.stepId,
-                        instructionIndex: stepContext?.instructionIndex,
-                        eventId: actionEventId,
-                        captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
-                        ...(annotatedScreenshotPath ? { screenshotPath: annotatedScreenshotPath } : {}),
-                        ...(context?.includeScreenshotDataUrl && annotatedScreenshotDataUrl ? { screenshotDataUrl: annotatedScreenshotDataUrl } : {})
-                    });
-                    if (trackAction) {
-                        trackAction(action);
-                    }
-                    if (shouldStop()) {
-                        return newResponseId;
-                    }
-                    if (action.type !== "wait") {
-                        await new Promise(resolve => setTimeout(resolve, this.stepDelayMs));
-                    }
+                if (toolResult.kind === "retry_instruction") {
+                    summary.retryInstruction = true;
+                    newResponseId = null;
+                    break;
                 }
-            }
-            if (shouldStop()) {
-                return newResponseId;
-            }
-            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-            emitDesktopDebug("device.screenshot", "device", {
-                runId: context?.runId,
-                stepId: stepContext?.stepId,
-                instructionIndex: stepContext?.instructionIndex
-            }, {
-                source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
-                callId: call_id,
-                width: this.session.deviceInfo?.scaled_width,
-                height: this.session.deviceInfo?.scaled_height,
-                base64Length: screenshotBase64.length
-            });
-            const screenshotEventId = randomUUID();
-            const captureSource = sawExplicitScreenshotAction ? CAPTURE_SOURCES.CALL_OUTPUT_EXPLICIT_ACTION : CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION;
-            const screenshotResult = await this.recordScreenshot(screenshotBase64, {
-                runId: context?.runId,
-                sessionId: context?.sessionId,
-                stepId: stepContext?.stepId,
-                instructionIndex: stepContext?.instructionIndex,
-                eventId: screenshotEventId,
-                captureSource
-            });
-            const screenshotPath = screenshotPathFromRecorderResult(screenshotResult);
-            addOutput({
-                type: 'info',
-                text: 'Capturing screen',
-                eventType: 'screenshot_captured',
-                actionType: 'screenshot',
-                ...eventMeta({
-                    payload: {
-                        eventId: screenshotEventId,
-                        captureSource,
-                        source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
-                        ...(typeof screenshotPath === "string" && screenshotPath ? { screenshotPath } : {}),
-                        ...(context?.includeScreenshotDataUrl
-                            ? { screenshotDataUrl: `data:image/png;base64,${screenshotBase64}` }
-                            : {})
-                    }
-                })
-            });
-            if (runtimeNote) {
-                const executedActionTypes = actionsToExecute
-                    .map((action) => action?.type)
-                    .filter(Boolean);
-                const droppedActionTypes = droppedActions
-                    .map((action) => action?.type)
-                    .filter(Boolean);
-                emitDesktopDebug("cua.strict_mode.truncation", scope, ids, {
-                    callId: call_id,
-                    executedCount: actionsToExecute.length,
-                    droppedCount: droppedActions.length,
-                    executedActionTypes,
-                    droppedActionTypes,
-                    reason: truncationReason,
-                    note: runtimeNote
+                if (toolResult.kind === "preserve_session_and_stop") {
+                    summary.preserveSessionMessages = true;
+                    newResponseId = null;
+                    break;
+                }
+                const nextTurnMessages = [toolResult.outputItem];
+                const nextTurnConfig = toolResult.nextTurnConfig ?? {};
+                if (Array.isArray(nextTurnConfig.followUpMessages) && nextTurnConfig.followUpMessages.length > 0) {
+                    nextTurnMessages.push(...nextTurnConfig.followUpMessages);
+                }
+                response = await this.requestNextTurn(nextTurnMessages, newResponseId, context, stepContext, {
+                    debugContextOverride: options.debugContextOverride,
+                    toolCapabilitiesOverride: nextTurnConfig.toolCapabilitiesOverride,
+                    includeComputerToolOverride: nextTurnConfig.includeComputerToolOverride,
                 });
+                newResponseId = response.id;
+                if (toolResult.kind === "close_turn") {
+                    break;
+                }
+                continue;
             }
-            const input = [{
-                    type: "computer_call_output",
-                    call_id,
-                    output: {
-                        type: "computer_screenshot",
-                        image_url: `data:image/png;base64,${screenshotBase64}`,
-                    },
-                    ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
-                }];
-            if (runtimeNote) {
-                input.push({
-                    role: "user",
-                    content: runtimeNote
-                });
+            const { call_id, actions } = actionableItem;
+            const actionBatchResult = await this.executeActionBatch(actions, currentScreenshot, shouldStop, context, stepContext, trackAction);
+            if (actionBatchResult.stopped) {
+                return finalize();
             }
+            const followUp = await this.buildActionFollowUp(call_id, pendingSafetyChecks, actionBatchResult.sawExplicitScreenshotAction, addOutput, eventMeta, context, stepContext);
             if (shouldStop()) {
-                return newResponseId;
+                return finalize();
             }
-            response = await this.sendCUARequest({
-                messages: input,
-                previousResponseId: newResponseId,
-                deviceInfo: this.session.deviceInfo,
-                debugContext: {
-                    scope,
-                    runId: context?.runId,
-                    sessionId: context?.sessionId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex
-                }
-            });
+            currentScreenshot = followUp.currentScreenshot ?? currentScreenshot;
+            response = await this.requestNextTurn(followUp.input, newResponseId, context, stepContext);
             newResponseId = response.id;
         }
         // ── At end, if last output was only reasoning ──
@@ -621,6 +382,6 @@ export class ExecutionEngine {
                 ...eventMeta()
             });
         }
-        return newResponseId;
+        return finalize();
     }
 }