@loadmill/droid-cua 2.10.0 → 2.10.1-rc.9

package/build/src/core/tool-handlers/utils.js

@@ -0,0 +1,80 @@
+/**
+ * Shared helpers for tool handlers. These are pure functions — no engine
+ * state. Anything that touches the engine (screenshots, network requests,
+ * the session) belongs in the handler files instead.
+ */
+export function parseJsonArguments(argumentsText) {
+    try {
+        return { ok: true, value: JSON.parse(argumentsText) };
+    }
+    catch {
+        return { ok: false, value: null };
+    }
+}
+export function parseStringToolArgument(argumentsText, key, toolName) {
+    const parsed = parseJsonArguments(argumentsText);
+    if (!parsed.ok || typeof parsed.value !== "object" || !parsed.value) {
+        return {
+            ok: false,
+            message: `Invalid ${toolName} arguments: expected a JSON object with a string "${key}" field.`,
+        };
+    }
+    const value = parsed.value[key];
+    if (typeof value !== "string" || !value.trim()) {
+        return {
+            ok: false,
+            message: `Invalid ${toolName} arguments: "${key}" must be a non-empty string.`,
+        };
+    }
+    return { ok: true, value: value.trim() };
+}
+export function parseStringArray(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value
+        .map((item) => (typeof item === "string" ? item.trim() : ""))
+        .filter(Boolean);
+}
+export function buildFunctionCallOutputItem(callId, output) {
+    return {
+        type: "function_call_output",
+        call_id: callId,
+        output,
+    };
+}
+export function buildTextToolOutputItem(callId, text) {
+    return buildFunctionCallOutputItem(callId, [{ type: "input_text", text }]);
+}
+export function buildPreviewOutputContent({ text, annotatedScreenshotBase64 = null }) {
+    const content = [{ type: "input_text", text }];
+    if (annotatedScreenshotBase64) {
+        content.push({
+            type: "input_image",
+            image_url: `data:image/png;base64,${annotatedScreenshotBase64}`,
+            detail: "high",
+        });
+    }
+    return content;
+}
+export function isDesignContext(context = null) {
+    return typeof context?.sessionId === "string" && context.sessionId.trim().length > 0;
+}
+export function buildDebugContext(context = null, stepContext = null) {
+    return {
+        scope: context?.sessionId ? "design" : "execution",
+        runId: context?.runId,
+        sessionId: context?.sessionId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+    };
+}
+export function buildNestedToolDebugContext(context, stepContext, toolName, callId) {
+    const debugContext = buildDebugContext(context, stepContext);
+    const baseChainId = debugContext.scope === "design"
+        ? debugContext.sessionId || "unknown"
+        : debugContext.runId || "unknown";
+    return {
+        ...debugContext,
+        chainId: `${debugContext.scope}:${baseChainId}:${toolName}:${callId}`,
+    };
+}

package/build/src/device/openai.js

@@ -34,6 +34,99 @@ const previewClickTool = {
         additionalProperties: false,
     },
 };
+const assertTool = {
+    type: "function",
+    name: "assert",
+    description: "Validate whether a condition is true on the current screen. Use this for instructions like assert, verify, validate, ensure, or confirm. Returns a pass/fail result for the requested condition.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            condition: { type: "string" },
+        },
+        required: ["condition"],
+        additionalProperties: false,
+    },
+};
+const navigateTool = {
+    type: "function",
+    name: "navigate",
+    description: "Navigate the active browser session directly to a target URL or page. Use this instead of trying to click the browser address bar or simulating URL entry.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            target: { type: "string" },
+        },
+        required: ["target"],
+        additionalProperties: false,
+    },
+};
+const loadmillTool = {
+    type: "function",
+    name: "loadmill",
+    description: "Run an explicit Loadmill command. Use this only when the instruction explicitly mentions Loadmill and asks for a Loadmill-specific action.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            command: { type: "string" },
+        },
+        required: ["command"],
+        additionalProperties: false,
+    },
+};
+const testDesignCompleteTool = {
+    type: "function",
+    name: "test_design_complete",
+    description: "Signal that design exploration is complete and hand off a structured summary for final test generation. Use this only after the requested flow has been successfully demonstrated or validated. After calling this, stop exploring.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            suggestedTestName: { type: "string" },
+            goalSummary: { type: "string" },
+            finalState: { type: "string" },
+            importantSteps: {
+                type: "array",
+                items: { type: "string" }
+            },
+            verificationPoints: {
+                type: "array",
+                items: { type: "string" }
+            },
+            notesForScript: { type: "string" },
+        },
+        required: [
+            "suggestedTestName",
+            "goalSummary",
+            "finalState",
+            "importantSteps",
+            "verificationPoints",
+            "notesForScript"
+        ],
+        additionalProperties: false,
+    },
+};
+export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, toolCapabilities = {}, } = {}) {
+    const tools = [];
+    if (aimAssistMode !== "off" && toolCapabilities.previewClick !== false) {
+        tools.push(previewClickTool);
+    }
+    if (toolCapabilities.assert !== false) {
+        tools.push(assertTool);
+    }
+    if (toolCapabilities.navigate) {
+        tools.push(navigateTool);
+    }
+    if (toolCapabilities.loadmill) {
+        tools.push(loadmillTool);
+    }
+    if (toolCapabilities.testDesignComplete) {
+        tools.push(testDesignCompleteTool);
+    }
+    return tools;
+}
 function appendInitialScreenshot(input, screenshotBase64) {
     // Most callers end with the active user message, but some recovery flows can
     // reset the conversation to system-only state before the next screenshot send.
@@ -91,7 +184,7 @@ function buildContextManagement(contextOptimizationEnabled, contextOptimizationT
     }
     return [{ type: "compaction", compact_threshold: contextOptimizationThreshold }];
 }
-export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, }) {
+export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, previousResponseId, input, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, }) {
     const contextManagement = buildContextManagement(contextOptimizationEnabled, contextOptimizationThreshold);
     // Keep Responses API storage enabled on every turn so execution can chain
     // reliably through previous_response_id across the whole run.
@@ -103,12 +196,10 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
         input,
         store: true,
         truncation: "auto",
-        tools: aimAssistMode === "off"
-            ? [{ type: "computer" }]
-            : [
-                { type: "computer" },
-                previewClickTool,
-            ],
+        tools: [
+            ...(includeComputerTool ? [{ type: "computer" }] : []),
+            ...buildFunctionTools({ aimAssistMode, toolCapabilities }),
+        ],
     };
 }
 /**
@@ -144,15 +235,17 @@ export async function reviseTestScript({ client, originalScript, revisionRequest
  *   debugContext?: unknown,
  * }} options
  */
-export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
+export async function sendCUARequest({ client, cuaModel, aimAssistMode = DEFAULT_AIM_ASSIST_MODE, includeComputerTool = true, contextOptimizationEnabled = true, contextOptimizationThreshold = DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, toolCapabilities = {}, messages, screenshotBase64, previousResponseId, callId, deviceInfo, debugContext, }) {
     const input = buildCuaInput({ messages, screenshotBase64, previousResponseId, callId });
     const requestParams = buildCuaRequestParams({
         cuaModel,
         aimAssistMode,
         previousResponseId,
         input,
+        includeComputerTool,
         contextOptimizationEnabled,
         contextOptimizationThreshold,
+        toolCapabilities,
     });
     const trace = cuaDebugTracer.startTurn({
         requestParams,

package/build/src/modes/design-mode-ink.js

@@ -1,4 +1,5 @@
 import { getScreenshotAsBase64 } from "../device/connection.js";
+import { CAPTURE_SOURCES } from "../core/capture-sources.js";
 import { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../core/prompts.js";
 import { saveTest } from "../test-store/test-manager.js";
 import { logger } from "../utils/logger.js";
@@ -35,6 +36,14 @@ export class DesignModeInk {
         const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
         this.baseDesignPrompt = designPrompt;
         this.session.setSystemPrompt(designPrompt);
+        this.engine.toolCapabilities = {
+            assert: true,
+            navigate: this.session?.deviceInfo?.platform === "browser",
+            loadmill: false,
+            testDesignComplete: true,
+        };
+        this.engine.baseSystemPrompt = designPrompt;
+        this.engine.isHeadlessMode = false;
         // Update UI
         if (this.context.setMode) {
             this.context.setMode('design');
@@ -182,14 +191,19 @@ export class DesignModeInk {
                 }
                 // Get screenshot and send to model
                 const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+                const currentScreenshot = await this.engine.recordScreenshot?.(screenshotBase64, {
+                    sessionId: this.context?.sessionId,
+                    captureSource: CAPTURE_SOURCES.DESIGN_INPUT,
+                });
                 const response = await this.sendCUARequest({
                     messages: this.session.messages,
                     screenshotBase64,
                     previousResponseId: this.session.previousResponseId,
                     deviceInfo: this.session.deviceInfo,
+                    toolCapabilities: this.engine.toolCapabilities,
                 });
                 // Run turn with action tracking and interruption checking
-                const newResponseId = await this.engine.runFullTurn(response, (action) => {
+                const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
                     // Check if user interrupted - stop immediately
                     if (this.inputQueue.length > 0) {
                         return true; // Signal to stop execution
@@ -205,7 +219,7 @@ export class DesignModeInk {
                         return true; // Stop for guidance
                     }
                     return false; // Continue execution
-                }, this.context);
+                }, this.context, null, { currentScreenshot });
                 this.session.updateResponseId(newResponseId);
                 this.consecutiveErrorCount = 0;
                 // Clear agent working status

package/build/src/modes/design-mode.js

@@ -1,5 +1,6 @@
 import readline from "readline";
 import { getScreenshotAsBase64 } from "../device/connection.js";
+import { CAPTURE_SOURCES } from "../core/capture-sources.js";
 import { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../core/prompts.js";
 import { saveTest } from "../test-store/test-manager.js";
 import { logger } from "../utils/logger.js";
@@ -32,6 +33,14 @@ export class DesignMode {
         const designPrompt = buildDesignModePrompt(this.session.deviceInfo, {});
         this.baseDesignPrompt = designPrompt;
         this.session.setSystemPrompt(designPrompt);
+        this.engine.toolCapabilities = {
+            assert: true,
+            navigate: this.session?.deviceInfo?.platform === "browser",
+            loadmill: false,
+            testDesignComplete: true,
+        };
+        this.engine.baseSystemPrompt = designPrompt;
+        this.engine.isHeadlessMode = false;
         console.log(`\n=== Design Mode: Creating test "${this.testName}" ===`);
         console.log("Describe what you want to test. The agent will explore autonomously.");
         console.log('When ready, type "generate the script" to create the test.');
@@ -146,14 +155,19 @@ export class DesignMode {
                 const keypressHandler = this.setupEscDetection();
                 // Get screenshot and send to model
                 const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+                const currentScreenshot = await this.engine.recordScreenshot?.(screenshotBase64, {
+                    sessionId: context?.sessionId,
+                    captureSource: CAPTURE_SOURCES.DESIGN_INPUT,
+                });
                 const response = await this.sendCUARequest({
                     messages: this.session.messages,
                     screenshotBase64,
                     previousResponseId: this.session.previousResponseId,
                     deviceInfo: this.session.deviceInfo,
+                    toolCapabilities: this.engine.toolCapabilities,
                 });
                 // Run turn with action tracking and interruption checking
-                const newResponseId = await this.engine.runFullTurn(response, (action) => {
+                const { responseId: newResponseId } = await this.engine.runFullTurn(response, (action) => {
                     // Check if user pressed ESC - stop immediately
                     if (this.escPressed) {
                         return true; // Signal to stop execution
@@ -170,7 +184,7 @@ export class DesignMode {
                         this.session._shouldStopForGuidance = true;
                     }
                     return false; // Continue execution
-                });
+                }, null, null, { currentScreenshot });
                 this.session.updateResponseId(newResponseId);
                 this.consecutiveErrorCount = 0;
                 // Cleanup ESC detection

package/build/src/modes/execution-mode.js

@@ -1,40 +1,13 @@
-import { randomUUID } from "node:crypto";
 import { CAPTURE_SOURCES } from "../core/capture-sources.js";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
-import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, navigateToUrl as navigateBrowserToUrl, } from "../browser/connection.js";
+import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, } from "../browser/connection.js";
 import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
-import { extractNavigateTarget, isNavigateInstruction } from "../browser/navigation.js";
 import { sendCUARequest } from "../device/openai.js";
-import { buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt } from "../core/prompts.js";
-import { isAssertion, extractAssertionPrompt, buildAssertionSystemPrompt, checkAssertionResult, handleAssertionFailure, handleAssertionSuccess, } from "../device/assertions.js";
-import { isLoadmillInstruction, extractLoadmillCommand, executeLoadmillInstruction, } from "../device/loadmill.js";
+import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
+import { isAssertion } from "../device/assertions.js";
 import { logger } from "../utils/logger.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { printCliOutput } from "../utils/console-output.js";
-export function buildExecutionRequestPayload({ instruction, isAssertionStep, messages, previousResponseId }) {
-    return {
-        messagesToSend: previousResponseId && !isAssertionStep
-            ? [{ role: "user", content: instruction }]
-            : messages,
-        previousResponseIdToSend: previousResponseId,
-    };
-}
-function screenshotPathFromRecorderResult(result) {
-    if (result && typeof result === "object" && !Array.isArray(result)) {
-        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
-            ? result.screenshotPath
-            : null;
-    }
-    return null;
-}
-function screenshotDataUrlFromRecorderResult(result) {
-    if (result && typeof result === "object" && !Array.isArray(result)) {
-        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
-            ? result.screenshotDataUrl
-            : null;
-    }
-    return null;
-}
 /**
  * Execution Mode - Run test scripts line-by-line
  * Each instruction is executed in isolation (messages cleared after each turn)
@@ -65,7 +38,15 @@ export class ExecutionMode {
             executionEngine.getCurrentPlatform || (this.isBrowserSession ? getBrowserCurrentPlatform : getCurrentPlatform);
         this.connectToBrowser = executionEngine.connectToBrowser || connectToBrowser;
         this.getBrowserInfo = executionEngine.getBrowserInfo || getBrowserInfo;
-        this.navigateToUrl = executionEngine.navigateToUrl || navigateBrowserToUrl;
+        // The Ink shell shares one engine across modes, so design mode may have
+        // changed these three fields. Reset them to the values execution needs.
+        this.engine.toolCapabilities = {
+            assert: true,
+            navigate: this.isBrowserSession,
+            loadmill: true,
+        };
+        this.engine.baseSystemPrompt = this.initialSystemText;
+        this.engine.isHeadlessMode = this.isHeadlessMode;
     }
     /**
      * Format duration in human-readable format (Xm Ys)
@@ -142,7 +123,7 @@ export class ExecutionMode {
                 eventType: 'instruction_started',
                 payload: {
                     instruction,
-                    isAssertion: isAssertion(instruction)
+                    isAssertion: isAssertion(instruction),
                 }
             });
             // Check for exit command
@@ -213,165 +194,28 @@ export class ExecutionMode {
      */
     async executeInstruction(instruction, context, retryCount = 0, stepContext = null) {
         const MAX_RETRIES = 10;
-        const addOutput = context.addOutput || printCliOutput;
-        if (isNavigateInstruction(instruction)) {
-            if (!this.isBrowserSession) {
-                const message = "navigate: instructions are only supported for web runs.";
-                this.emit(addOutput, "error", message, context, stepContext, {
-                    eventType: "error",
-                    payload: {
-                        instructionType: "navigate",
-                        message
-                    }
-                });
-                return { success: false, error: message };
-            }
-            const navigationTarget = extractNavigateTarget(instruction);
-            this.session.addToTranscript(`[Navigate] ${navigationTarget}`);
-            emitDesktopDebug("runner.navigate.started", "execution", {
-                runId: context?.runId,
-                stepId: stepContext?.stepId,
-                instructionIndex: stepContext?.instructionIndex
-            }, {
-                instruction,
-                target: navigationTarget
-            });
-            this.emit(addOutput, "system", `Navigating to ${navigationTarget}...`, context, stepContext, {
-                eventType: "system_message",
-                payload: {
-                    instructionType: "navigate",
-                    target: navigationTarget,
-                }
-            });
-            try {
-                const normalizedUrl = await this.navigateToUrl(this.session.browserSessionId, navigationTarget);
-                const browserInfo = normalizeBrowserInfoForSession(await this.getBrowserInfo(this.session.browserSessionId));
-                this.session.browserInfo = browserInfo;
-                this.session.deviceInfo = browserInfo;
-                this.session.currentUrl = browserInfo.currentUrl || normalizedUrl;
-                this.session.browserName = browserInfo.browserName || this.session.currentUrl;
-                this.session.deviceName = this.session.browserName;
-                const transcriptContext = this.session.getTranscriptText();
-                const navigationRecoveryPrompt = buildExecutionNavigationRecoveryPrompt({
-                    basePrompt: this.initialSystemText,
-                    transcript: transcriptContext,
-                    navigatedUrl: this.session.currentUrl,
-                });
-                this.session.installRecoverySystemPrompt(navigationRecoveryPrompt);
-                emitDesktopDebug("runner.navigate.completed", "execution", {
-                    runId: context?.runId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex
-                }, {
-                    instruction,
-                    target: navigationTarget,
-                    url: normalizedUrl
-                });
-                this.emit(addOutput, "success", `Navigation complete: ${normalizedUrl}`, context, stepContext, {
-                    eventType: "system_message",
-                    payload: {
-                        instructionType: "navigate",
-                        url: normalizedUrl,
-                    }
-                });
-                return { success: true };
-            }
-            catch (err) {
-                const message = err instanceof Error ? err.message : "Navigation failed.";
-                logger.error("Execution navigation error", {
-                    instruction,
-                    retryCount,
-                    message,
-                });
-                emitDesktopDebug("runner.navigate.failed", "execution", {
-                    runId: context?.runId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex
-                }, {
-                    instruction,
-                    target: navigationTarget,
-                    message
-                });
-                this.emit(addOutput, "error", `Navigation failed: ${message}`, context, stepContext, {
-                    eventType: "error",
-                    payload: {
-                        instructionType: "navigate",
-                        message,
-                    }
-                });
-                return { success: false, error: message };
-            }
-        }
-        // ── Check for Loadmill instruction ──
-        if (isLoadmillInstruction(instruction)) {
-            const loadmillCommand = extractLoadmillCommand(instruction);
-            this.session.addToTranscript(`[Loadmill] ${loadmillCommand}`);
-            const result = await executeLoadmillInstruction(loadmillCommand, this.isHeadlessMode, context, stepContext);
-            // Handle retry request from interactive mode
-            if (result.retry) {
-                this.stats.retryCount++;
-                return await this.executeInstruction(instruction, context, 0, stepContext);
-            }
-            return result;
-        }
-        // ── Check for assertion ──
-        const isAssertionStep = isAssertion(instruction);
-        const assertionEventId = isAssertionStep ? randomUUID() : null;
-        let assertionPrompt = null;
-        const transcriptStartIndex = this.session.transcript.length;
-        if (isAssertionStep) {
-            assertionPrompt = extractAssertionPrompt(instruction);
-            const assertionSystemPrompt = buildAssertionSystemPrompt(this.initialSystemText, assertionPrompt);
-            this.session.clearMessages();
-            this.session.addMessage("system", assertionSystemPrompt);
-            this.session.addToTranscript(`[Assertion] ${assertionPrompt}`);
-            this.session.addMessage("user", `Validate this assertion: ${assertionPrompt}`);
-        }
-        else {
-            this.session.addToTranscript(`[User] ${instruction}`);
-            this.session.addMessage("user", instruction);
-        }
+        this.session.addToTranscript(`[User] ${instruction}`);
+        this.session.addMessage("user", instruction);
         try {
             const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-            const screenshotResult = await this.engine.recordScreenshot?.(screenshotBase64, {
+            const currentScreenshot = await this.engine.recordScreenshot?.(screenshotBase64, {
                 runId: context?.runId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
-                ...(assertionEventId ? { eventId: assertionEventId } : {}),
-                captureSource: isAssertionStep ? CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION : CAPTURE_SOURCES.INSTRUCTION_INPUT
-            });
-            const assertionScreenshotPath = isAssertionStep ? screenshotPathFromRecorderResult(screenshotResult) : null;
-            const assertionScreenshotDataUrl = isAssertionStep
-                ? screenshotDataUrlFromRecorderResult(screenshotResult) ||
-                    (context?.includeScreenshotDataUrl && assertionScreenshotPath
-                        ? `data:image/png;base64,${screenshotBase64}`
-                        : null)
-                : null;
-            const assertionEventPayload = isAssertionStep
-                ? {
-                    eventId: assertionEventId,
-                    ...(assertionScreenshotPath ? { screenshotPath: assertionScreenshotPath } : {}),
-                    ...(assertionScreenshotDataUrl ? { screenshotDataUrl: assertionScreenshotDataUrl } : {})
-                }
-                : {};
-            // Assertions rely on the prior response chain for earlier execution context.
-            const { messagesToSend, previousResponseIdToSend } = buildExecutionRequestPayload({
-                instruction,
-                isAssertionStep,
-                messages: this.session.messages,
-                previousResponseId: this.session.previousResponseId
+                captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT
             });
             const response = await this.sendCUARequest({
-                messages: messagesToSend,
+                messages: this.session.messages,
                 screenshotBase64,
-                previousResponseId: previousResponseIdToSend,
+                previousResponseId: this.session.previousResponseId,
                 deviceInfo: this.session.deviceInfo,
                 debugContext: {
                     scope: "execution",
                     runId: context?.runId,
                     stepId: stepContext?.stepId,
                     instructionIndex: stepContext?.instructionIndex
-                }
+                },
+                toolCapabilities: this.engine.toolCapabilities,
             });
             // Track actions for stats
             const trackAction = (action = null) => {
@@ -380,65 +224,29 @@ export class ExecutionMode {
                 }
                 return this.shouldStop;
             };
-            const newResponseId = await this.engine.runFullTurn(response, trackAction, context, stepContext);
-            this.session.updateResponseId(newResponseId);
-            // ── Check assertion result ──
-            if (isAssertionStep) {
-                const assertionTranscript = this.session.transcript.slice(transcriptStartIndex);
-                const result = checkAssertionResult(assertionTranscript);
-                if (result.failed) {
-                    handleAssertionFailure(assertionPrompt, assertionTranscript, false, // Never exit process - we'll always prompt the user in interactive mode
-                    context, stepContext, assertionEventPayload);
-                    // In headless mode, exit immediately on assertion failure
-                    if (this.isHeadlessMode) {
-                        this.stats.assertionsFailed++;
-                        return { success: false, error: `Assertion failed: ${assertionPrompt}` };
-                    }
-                    // Interactive mode - ask user what to do
-                    this.emit(addOutput, 'system', 'What would you like to do? (retry/skip/stop)', context, stepContext, {
-                        eventType: 'input_request',
-                        payload: {
-                            options: ['retry', 'skip', 'stop']
-                        }
-                    });
-                    // Wait for user input
-                    const userChoice = await new Promise((resolve) => {
-                        if (context?.waitForUserInput) {
-                            context.waitForUserInput().then(resolve);
-                        }
-                        else {
-                            // Fallback if waitForUserInput not available
-                            resolve('stop');
-                        }
-                    });
-                    const choice = userChoice.toLowerCase().trim();
-                    if (choice === 'retry' || choice === 'r') {
-                        // Retry the same instruction by recursing
-                        this.stats.retryCount++;
-                        return await this.executeInstruction(instruction, context, 0, stepContext);
-                    }
-                    else if (choice === 'skip' || choice === 's') {
-                        // Continue to next instruction
-                        this.stats.assertionsFailed++;
-                        this.emit(addOutput, 'info', 'Skipping failed assertion and continuing...', context, stepContext, {
-                            eventType: 'system_message'
-                        });
-                    }
-                    else {
-                        // Stop execution
-                        this.stats.assertionsFailed++;
-                        return { success: false, error: `Assertion failed: ${assertionPrompt}` };
-                    }
-                }
-                else if (result.passed) {
-                    this.stats.assertionsPassed++;
-                    handleAssertionSuccess(assertionPrompt, context, stepContext, assertionEventPayload);
+            const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, trackAction, context, stepContext, { currentScreenshot });
+            if (turnSummary.assertionStatus === "passed") {
+                this.stats.assertionsPassed++;
+            }
+            else if (turnSummary.assertionStatus === "failed" ||
+                turnSummary.assertionStatus === "failed_skipped") {
+                this.stats.assertionsFailed++;
+            }
+            if (turnSummary.retryInstruction) {
+                this.stats.retryCount++;
+                this.session.updateResponseId(turnSummary.responseId);
+                if (!turnSummary.preserveSessionMessages) {
+                    this.session.clearMessages();
                 }
+                return await this.executeInstruction(instruction, context, 0, stepContext);
+            }
+            if (turnSummary.failed) {
+                return { success: false, error: turnSummary.error || "Instruction failed." };
+            }
+            this.session.updateResponseId(turnSummary.responseId ?? newResponseId);
+            if (!turnSummary.preserveSessionMessages) {
+                this.session.clearMessages();
             }
-            // Clear messages after each turn but KEEP the response chain for context
-            // When continuing with previousResponseId, only send new user message (not system)
-            this.session.clearMessages();
-            // Note: we keep previousResponseId to maintain context across the test
             return { success: true };
         }
         catch (err) {