@loadmill/droid-cua 2.9.0 → 2.10.1-rc.10

package/build/src/core/prompts.js

@@ -1,3 +1,3 @@
 export { buildBaseSystemPrompt } from "../prompts/base.js";
-export { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../prompts/design.js";
+export { buildDesignModePrompt, buildDesignRecoveryPrompt, buildDesignScriptGenerationPrompt, } from "../prompts/design.js";
 export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";

package/build/src/core/runtime-config.js

@@ -35,10 +35,9 @@ export function resolveRuntimeCredentials({ openAiApiKey = null, loadmillApiToke
         loadmillBaseUrl: trimNullableString(loadmillBaseUrl),
     };
 }
-export function resolveRuntimeConfig({ cuaModel = null, strictMode = false, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
+export function resolveRuntimeConfig({ cuaModel = null, stepDelayMs = undefined, contextOptimizationEnabled = undefined, contextOptimizationThreshold = undefined, promptCustomizations = null, appContext = null, } = {}) {
     return {
         cuaModel: resolveCuaModel(cuaModel),
-        strictMode: strictMode === true,
         stepDelayMs: normalizeStepDelayMs(Number(stepDelayMs)),
         contextOptimizationEnabled: normalizeContextOptimizationEnabled(contextOptimizationEnabled),
         contextOptimizationThreshold: normalizeContextOptimizationThreshold(contextOptimizationThreshold),

package/build/src/core/screenshot-store.js

@@ -0,0 +1,153 @@
+import path from "path";
+import { writeFile } from "fs/promises";
+import { emitDesktopDebug } from "../utils/desktop-debug.js";
+import { annotateScreenshotArtifact } from "../utils/run-screenshot-recorder.js";
+function normalizeStringList(value) {
+    if (!Array.isArray(value)) {
+        return typeof value === "string" && value.trim() ? [value] : [];
+    }
+    return value.flatMap((item) => {
+        if (typeof item === "string" && item.trim()) {
+            return [item];
+        }
+        if (item && typeof item === "object") {
+            return normalizeStringList(item.artifactPaths);
+        }
+        return [];
+    });
+}
+function normalizeRecorderResult(result, screenshotBase64) {
+    if (!result) {
+        return null;
+    }
+    if (typeof result === "string") {
+        return {
+            base64: screenshotBase64,
+            artifactPaths: [result],
+            annotationSourcePaths: [result],
+            screenshotPath: result,
+            screenshotDataUrl: null,
+        };
+    }
+    if (Array.isArray(result)) {
+        const normalizedItems = result
+            .map((item) => normalizeRecorderResult(item, screenshotBase64))
+            .filter(Boolean);
+        const artifactPaths = normalizedItems.flatMap((item) => item.artifactPaths);
+        const annotationSourcePaths = normalizedItems.flatMap((item) => item.annotationSourcePaths);
+        return {
+            base64: screenshotBase64,
+            artifactPaths,
+            annotationSourcePaths,
+            screenshotPath: normalizedItems.find((item) => item.screenshotPath)?.screenshotPath ?? artifactPaths[0] ?? null,
+            screenshotDataUrl: normalizedItems.find((item) => item.screenshotDataUrl)?.screenshotDataUrl ?? null,
+        };
+    }
+    if (typeof result === "object") {
+        const artifactPaths = normalizeStringList(result.artifactPaths);
+        const annotationSourcePaths = Array.isArray(result.annotationSourcePaths)
+            ? normalizeStringList(result.annotationSourcePaths)
+            : artifactPaths;
+        const screenshotPath = typeof result.screenshotPath === "string" && result.screenshotPath.trim()
+            ? result.screenshotPath
+            : artifactPaths[0] ?? null;
+        const screenshotDataUrl = typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
+            ? result.screenshotDataUrl
+            : null;
+        return {
+            base64: screenshotBase64,
+            artifactPaths,
+            annotationSourcePaths,
+            screenshotPath,
+            screenshotDataUrl,
+        };
+    }
+    return null;
+}
+function debugScope(metadata = {}) {
+    return typeof metadata.sessionId === "string" ? "design" : "execution";
+}
+function debugIds(metadata = {}) {
+    if (debugScope(metadata) === "design") {
+        return {
+            sessionId: metadata.sessionId,
+            stepId: metadata.stepId,
+            instructionIndex: metadata.instructionIndex,
+        };
+    }
+    return {
+        runId: metadata.runId,
+        stepId: metadata.stepId,
+        instructionIndex: metadata.instructionIndex,
+    };
+}
+export class ScreenshotStore {
+    constructor({ recordScreenshots = false, screenshotDir = null, screenshotRecorder = null } = {}) {
+        this.recordScreenshots = recordScreenshots;
+        this.screenshotDir = screenshotDir;
+        this.screenshotRecorder = screenshotRecorder;
+        this.reportedWriteError = false;
+        this.reportedAnnotateError = false;
+    }
+    async saveScreenshot(screenshotBase64, metadata = {}) {
+        if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
+            return null;
+        }
+        try {
+            if (this.screenshotRecorder?.saveScreenshot) {
+                const result = await this.screenshotRecorder.saveScreenshot(screenshotBase64, metadata);
+                return normalizeRecorderResult(result, screenshotBase64);
+            }
+            if (this.recordScreenshots && this.screenshotDir) {
+                const framePath = path.join(this.screenshotDir, `frame_${String(Date.now())}.png`);
+                await writeFile(framePath, Buffer.from(screenshotBase64, "base64"));
+                return normalizeRecorderResult(framePath, screenshotBase64);
+            }
+        }
+        catch (error) {
+            this.reportWriteError(error, metadata);
+        }
+        return null;
+    }
+    async annotateScreenshot(savedScreenshot, marker = {}, metadata = {}) {
+        const sourcePaths = Array.isArray(savedScreenshot?.annotationSourcePaths)
+            ? savedScreenshot.annotationSourcePaths.filter((filePath) => typeof filePath === "string" && filePath.trim())
+            : [];
+        if (sourcePaths.length === 0) {
+            return null;
+        }
+        try {
+            const sourcePath = sourcePaths[0];
+            if (sourcePath && this.screenshotRecorder?.saveAnnotatedScreenshot) {
+                const result = await this.screenshotRecorder.saveAnnotatedScreenshot(sourcePath, marker, metadata);
+                return normalizeRecorderResult(result, savedScreenshot.base64 ?? null);
+            }
+            await Promise.all(sourcePaths.map((filePath) => annotateScreenshotArtifact(filePath, marker)));
+            return null;
+        }
+        catch (error) {
+            this.reportAnnotateError(error, marker, metadata);
+            return null;
+        }
+    }
+    reportWriteError(error, metadata = {}) {
+        if (this.reportedWriteError)
+            return;
+        this.reportedWriteError = true;
+        emitDesktopDebug("run.screenshot_write.error", debugScope(metadata), debugIds(metadata), {
+            captureSource: metadata.captureSource ?? null,
+            message: error instanceof Error ? error.message : "Failed to persist screenshot",
+        });
+    }
+    reportAnnotateError(error, marker = {}, metadata = {}) {
+        if (this.reportedAnnotateError)
+            return;
+        this.reportedAnnotateError = true;
+        emitDesktopDebug("run.screenshot_annotate.error", debugScope(metadata), debugIds(metadata), {
+            actionType: metadata.actionType ?? null,
+            x: marker?.x ?? null,
+            y: marker?.y ?? null,
+            message: error instanceof Error ? error.message : "Failed to annotate screenshot",
+        });
+    }
+}

package/build/src/core/tool-handlers/assert.js

@@ -0,0 +1,155 @@
+import { randomUUID } from "node:crypto";
+import { buildAssertionSystemPrompt, checkAssertionResult, extractFailureDetails, handleAssertionFailure, handleAssertionSuccess, } from "../../device/assertions.js";
+import { CAPTURE_SOURCES } from "../capture-sources.js";
+import { printCliOutput } from "../../utils/console-output.js";
+import { buildNestedToolDebugContext, buildTextToolOutputItem, isDesignContext, parseStringToolArgument, } from "./utils.js";
+export async function handleAssertionToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    const parsed = parseStringToolArgument(argumentsText, "condition", "assert");
+    if (!parsed.ok) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, parsed.message),
+        };
+    }
+    const assertionPrompt = parsed.value;
+    const transcriptStartIndex = session.transcript.length;
+    session.addToTranscript(`[Assertion] ${assertionPrompt}`);
+    const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
+    const assertionEventId = randomUUID();
+    const assertionScreenshot = await engine.recordScreenshot(screenshotBase64, {
+        runId: context?.runId,
+        sessionId: context?.sessionId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        eventId: assertionEventId,
+        captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
+    });
+    const assertionScreenshotPath = assertionScreenshot?.screenshotPath ?? null;
+    const assertionScreenshotDataUrl = assertionScreenshot?.screenshotDataUrl ||
+        (context?.includeScreenshotDataUrl && assertionScreenshotPath
+            ? `data:image/png;base64,${screenshotBase64}`
+            : null);
+    const assertionEventPayload = {
+        eventId: assertionEventId,
+        ...(assertionScreenshotPath ? { screenshotPath: assertionScreenshotPath } : {}),
+        ...(assertionScreenshotDataUrl ? { screenshotDataUrl: assertionScreenshotDataUrl } : {}),
+    };
+    const assertionSystemPrompt = buildAssertionSystemPrompt(engine.baseSystemPrompt || session.systemPrompt || "", assertionPrompt);
+    const nestedDebugContext = buildNestedToolDebugContext(context, stepContext, "assert", callId);
+    const assertionResponse = await engine.sendCUARequest({
+        messages: [
+            { role: "system", content: assertionSystemPrompt },
+            { role: "user", content: `Validate this assertion: ${assertionPrompt}` },
+        ],
+        screenshotBase64,
+        previousResponseId: null,
+        deviceInfo: session.deviceInfo,
+        debugContext: nestedDebugContext,
+        toolCapabilities: {
+            assert: false,
+            navigate: false,
+            loadmill: false,
+        },
+    });
+    // Nested assertion turn: we discard the returned summary because the
+    // parent turn (the one that dispatched the assert tool call) is what
+    // the caller will read.
+    const nestedTurn = await engine.runFullTurn(assertionResponse, () => false, context, stepContext, {
+        suppressAssistantOutput: true,
+        debugContextOverride: nestedDebugContext,
+        currentScreenshot: assertionScreenshot,
+    });
+    const currentScreenshot = nestedTurn.currentScreenshot ?? assertionScreenshot;
+    const assertionTranscript = session.transcript.slice(transcriptStartIndex);
+    const result = checkAssertionResult(assertionTranscript);
+    if (isDesignContext(context)) {
+        const details = result.passed
+            ? "Assertion passed."
+            : `Assertion failed: ${extractFailureDetails(assertionTranscript)}`;
+        if (result.passed) {
+            handleAssertionSuccess(assertionPrompt, context, stepContext, assertionEventPayload);
+        }
+        else {
+            const addOutput = context?.addOutput || printCliOutput;
+            addOutput({
+                type: "error",
+                text: `Assertion failed: ${assertionPrompt}`,
+                eventType: "assertion_result",
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                payload: {
+                    assertion: assertionPrompt,
+                    passed: false,
+                    details: extractFailureDetails(assertionTranscript),
+                    ...assertionEventPayload,
+                },
+            });
+        }
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, result.passed
+                ? `ASSERTION RESULT: PASS\n${details}`
+                : `ASSERTION RESULT: FAIL\n${details}`),
+            currentScreenshot,
+        };
+    }
+    if (result.failed) {
+        handleAssertionFailure(assertionPrompt, assertionTranscript, false, context, stepContext, assertionEventPayload);
+        if (engine.isHeadlessMode) {
+            return {
+                kind: "fail",
+                error: `Assertion failed: ${assertionPrompt}`,
+                summaryUpdates: { assertionStatus: "failed" },
+            };
+        }
+        const addOutput = context?.addOutput || printCliOutput;
+        addOutput({
+            type: "system",
+            text: "What would you like to do? (retry/skip/stop)",
+            eventType: "input_request",
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+            payload: {
+                options: ["retry", "skip", "stop"],
+            },
+        });
+        const userChoice = await new Promise((resolve) => {
+            if (context?.waitForUserInput) {
+                context.waitForUserInput().then(resolve);
+            }
+            else {
+                resolve("stop");
+            }
+        });
+        const choice = String(userChoice).toLowerCase().trim();
+        if (choice === "retry" || choice === "r") {
+            return {
+                kind: "retry_instruction",
+                summaryUpdates: { assertionStatus: "retry" },
+            };
+        }
+        if (choice === "skip" || choice === "s") {
+            return {
+                kind: "close_turn",
+                outputItem: buildTextToolOutputItem(callId, `ASSERTION RESULT: FAIL\n${extractFailureDetails(assertionTranscript)}`),
+                summaryUpdates: { assertionStatus: "failed_skipped" },
+                currentScreenshot,
+            };
+        }
+        return {
+            kind: "fail",
+            error: `Assertion failed: ${assertionPrompt}`,
+            summaryUpdates: { assertionStatus: "failed" },
+        };
+    }
+    handleAssertionSuccess(assertionPrompt, context, stepContext, assertionEventPayload);
+    return {
+        kind: "close_turn",
+        outputItem: buildTextToolOutputItem(callId, "ASSERTION RESULT: PASS\nAssertion passed."),
+        summaryUpdates: { assertionStatus: "passed" },
+        currentScreenshot,
+    };
+}

package/build/src/core/tool-handlers/loadmill.js

@@ -0,0 +1,29 @@
+import { buildTextToolOutputItem, parseStringToolArgument } from "./utils.js";
+export async function handleLoadmillToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    const parsed = parseStringToolArgument(argumentsText, "command", "loadmill");
+    if (!parsed.ok) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, parsed.message),
+        };
+    }
+    const loadmillCommand = parsed.value;
+    session.addToTranscript(`[Loadmill] ${loadmillCommand}`);
+    const result = await engine.executeLoadmillInstruction(loadmillCommand, engine.isHeadlessMode, context, stepContext);
+    if (result.retry) {
+        return {
+            kind: "retry_instruction",
+        };
+    }
+    if (!result.success) {
+        return {
+            kind: "fail",
+            error: result.error || `Loadmill command failed: ${loadmillCommand}`,
+        };
+    }
+    return {
+        kind: "close_turn",
+        outputItem: buildTextToolOutputItem(callId, "Loadmill command completed successfully."),
+    };
+}

package/build/src/core/tool-handlers/navigate.js

@@ -0,0 +1,110 @@
+import { normalizeBrowserInfoForSession } from "../../browser/normalize.js";
+import { buildExecutionNavigationRecoveryPrompt } from "../../prompts/execution.js";
+import { emitDesktopDebug } from "../../utils/desktop-debug.js";
+import { printCliOutput } from "../../utils/console-output.js";
+import { buildDebugContext, buildFunctionCallOutputItem, buildPreviewOutputContent, buildTextToolOutputItem, isDesignContext, parseStringToolArgument, } from "./utils.js";
+export async function handleNavigateToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    const parsed = parseStringToolArgument(argumentsText, "target", "navigate");
+    if (!parsed.ok) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, parsed.message),
+        };
+    }
+    if (session?.targetKind !== "browser" && session?.deviceInfo?.platform !== "browser") {
+        const message = "navigate tool is only supported for web runs.";
+        const addOutput = context?.addOutput || printCliOutput;
+        addOutput({
+            type: "error",
+            text: message,
+            eventType: "error",
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+            payload: {
+                instructionType: "navigate",
+                message,
+            },
+        });
+        return {
+            kind: "fail",
+            error: message,
+        };
+    }
+    const navigationTarget = parsed.value;
+    const addOutput = context?.addOutput || null;
+    session.addToTranscript(`[Navigate] ${navigationTarget}`);
+    emitDesktopDebug("runner.navigate.started", isDesignContext(context) ? "design" : "execution", buildDebugContext(context, stepContext), {
+        instruction: navigationTarget,
+        target: navigationTarget,
+    });
+    try {
+        const normalizedUrl = await engine.navigateToUrl(session.browserSessionId, navigationTarget);
+        const browserInfo = normalizeBrowserInfoForSession(await engine.getBrowserInfo(session.browserSessionId));
+        session.browserInfo = browserInfo;
+        session.deviceInfo = browserInfo;
+        session.currentUrl = browserInfo.currentUrl || normalizedUrl;
+        session.browserName = browserInfo.browserName || session.currentUrl;
+        session.deviceName = session.browserName;
+        emitDesktopDebug("runner.navigate.completed", isDesignContext(context) ? "design" : "execution", buildDebugContext(context, stepContext), {
+            instruction: navigationTarget,
+            target: navigationTarget,
+            url: normalizedUrl,
+        });
+        if (addOutput) {
+            addOutput({
+                type: "action",
+                text: `Navigate to ${normalizedUrl}`,
+                eventType: "tool_call",
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                payload: {
+                    platform: session.deviceInfo?.platform,
+                    toolName: "navigate",
+                    target: navigationTarget,
+                    url: normalizedUrl,
+                },
+            });
+        }
+        if (isDesignContext(context)) {
+            const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
+            return {
+                kind: "continue",
+                outputItem: buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                    text: `Navigation complete: ${normalizedUrl}`,
+                    annotatedScreenshotBase64: screenshotBase64,
+                })),
+            };
+        }
+        const transcriptContext = session.getTranscriptText();
+        const navigationRecoveryPrompt = buildExecutionNavigationRecoveryPrompt({
+            basePrompt: engine.baseSystemPrompt,
+            transcript: transcriptContext,
+            navigatedUrl: session.currentUrl,
+        });
+        session.installRecoverySystemPrompt(navigationRecoveryPrompt);
+        return {
+            kind: "preserve_session_and_stop",
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : "Navigation failed.";
+        emitDesktopDebug("runner.navigate.failed", isDesignContext(context) ? "design" : "execution", buildDebugContext(context, stepContext), {
+            instruction: navigationTarget,
+            target: navigationTarget,
+            message,
+        });
+        if (isDesignContext(context)) {
+            return {
+                kind: "continue",
+                outputItem: buildTextToolOutputItem(callId, `Navigation failed: ${message}`),
+            };
+        }
+        return {
+            kind: "fail",
+            error: message,
+        };
+    }
+}

package/build/src/core/tool-handlers/preview-click.js

@@ -0,0 +1,112 @@
+import { randomUUID } from "node:crypto";
+import { createAnnotatedScreenshotFromBase64 } from "../../utils/run-screenshot-recorder.js";
+import { CAPTURE_SOURCES } from "../capture-sources.js";
+import { buildFunctionCallOutputItem, buildPreviewOutputContent, } from "./utils.js";
+function buildPreviewClickValidationError(argumentsText) {
+    if (!argumentsText || typeof argumentsText !== "string") {
+        return "Invalid preview_click arguments: expected a JSON object with numeric x and y.";
+    }
+    return `Invalid preview_click arguments: ${argumentsText}`;
+}
+function parsePreviewClickArguments(argumentsText) {
+    let parsed;
+    try {
+        parsed = JSON.parse(argumentsText);
+    }
+    catch {
+        return {
+            ok: false,
+            message: buildPreviewClickValidationError("arguments could not be parsed as JSON; expected numeric x and y."),
+        };
+    }
+    const x = parsed?.x;
+    const y = parsed?.y;
+    if (!Number.isFinite(x) || !Number.isFinite(y)) {
+        return {
+            ok: false,
+            message: buildPreviewClickValidationError("x and y must both be numeric."),
+        };
+    }
+    return { ok: true, x, y };
+}
+function buildPreviewOutputText({ requestedX, requestedY, renderedX, renderedY, wasClamped }) {
+    const baseText = `Marker placed at (${renderedX}, ${renderedY}) on the current screen. The click has not been executed.`;
+    if (!wasClamped)
+        return baseText;
+    return `${baseText} Requested point (${requestedX}, ${requestedY}) was off-screen; marker rendered at (${renderedX}, ${renderedY}).`;
+}
+function continueWith(outputItem) {
+    return { kind: "continue", outputItem };
+}
+export async function handlePreviewClick({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    const parsedArguments = parsePreviewClickArguments(argumentsText);
+    if (!parsedArguments.ok) {
+        return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({ text: parsedArguments.message })));
+    }
+    try {
+        const screenshotBase64 = await engine.getScreenshotAsBase64(session.deviceId, session.deviceInfo);
+        const previewResult = await createAnnotatedScreenshotFromBase64(screenshotBase64, {
+            x: parsedArguments.x,
+            y: parsedArguments.y,
+            style: "preview",
+        });
+        if (!previewResult?.screenshotBase64 || !previewResult?.marker) {
+            return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                text: "preview_click failed: could not annotate the current screenshot.",
+            })));
+        }
+        const previewEventId = randomUUID();
+        const previewScreenshot = await engine.recordScreenshot(previewResult.screenshotBase64, {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex,
+            callId,
+            eventId: previewEventId,
+            captureSource: CAPTURE_SOURCES.PREVIEW_CLICK,
+        });
+        const previewText = buildPreviewOutputText({
+            requestedX: previewResult.marker.requestedX,
+            requestedY: previewResult.marker.requestedY,
+            renderedX: previewResult.marker.x,
+            renderedY: previewResult.marker.y,
+            wasClamped: previewResult.marker.wasClamped,
+        });
+        const addOutput = context?.addOutput || null;
+        if (addOutput) {
+            const previewScreenshotDataUrl = context?.includeScreenshotDataUrl
+                ? `data:image/png;base64,${previewResult.screenshotBase64}`
+                : null;
+            addOutput({
+                type: "action",
+                text: "Click",
+                eventType: "tool_call",
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                payload: {
+                    platform: session.deviceInfo?.platform,
+                    toolName: "preview_click",
+                    captureSource: CAPTURE_SOURCES.PREVIEW_CLICK,
+                    requestedX: previewResult.marker.requestedX,
+                    requestedY: previewResult.marker.requestedY,
+                    renderedX: previewResult.marker.x,
+                    renderedY: previewResult.marker.y,
+                    wasClamped: previewResult.marker.wasClamped,
+                    ...(previewScreenshot?.screenshotPath ? { screenshotPath: previewScreenshot.screenshotPath } : {}),
+                    ...(previewScreenshotDataUrl ? { screenshotDataUrl: previewScreenshotDataUrl } : {}),
+                },
+            });
+        }
+        return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+            text: previewText,
+            annotatedScreenshotBase64: previewResult.screenshotBase64,
+        })));
+    }
+    catch (error) {
+        return continueWith(buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+            text: `preview_click failed: ${error instanceof Error ? error.message : "unable to capture or annotate the current screen."}`,
+        })));
+    }
+}

package/build/src/core/tool-handlers/test-design-complete.js

@@ -0,0 +1,93 @@
+import { buildDesignScriptGenerationPrompt } from "../../prompts/design.js";
+import { printCliOutput } from "../../utils/console-output.js";
+import { buildTextToolOutputItem, isDesignContext, parseJsonArguments, parseStringArray, } from "./utils.js";
+// Renders the summary as the tool-call output the model sees right after
+// `test_design_complete`. Distinct from buildDesignScriptGenerationPrompt
+// (which goes in a follow-up user message): different headers, different
+// missing-field fallbacks ("(choose automatically)" here vs. "(choose a
+// short kebab-case name)" there).
+function formatDesignCompletionSummary(summary) {
+    const lines = [
+        "TEST DESIGN COMPLETE",
+        `Suggested test name: ${summary.suggestedTestName || "(choose automatically)"}`,
+        `Goal summary: ${summary.goalSummary}`,
+        `Final state: ${summary.finalState}`,
+        "Important steps:",
+        ...(summary.importantSteps.length > 0
+            ? summary.importantSteps.map((item) => `- ${item}`)
+            : ["- (none provided)"]),
+        "Verification points:",
+        ...(summary.verificationPoints.length > 0
+            ? summary.verificationPoints.map((item) => `- ${item}`)
+            : ["- (none provided)"]),
+    ];
+    if (summary.notesForScript) {
+        lines.push("Notes for script:");
+        lines.push(summary.notesForScript);
+    }
+    return lines.join("\n");
+}
+export async function handleTestDesignCompleteToolCall({ engine, callId, argumentsText, context = null, stepContext = null }) {
+    const session = engine.session;
+    if (!isDesignContext(context)) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, "test_design_complete is only supported in design mode."),
+        };
+    }
+    const parsed = parseJsonArguments(argumentsText);
+    if (!parsed.ok || !parsed.value || typeof parsed.value !== "object") {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, "Invalid test_design_complete arguments: expected a JSON object."),
+        };
+    }
+    const summary = {
+        suggestedTestName: typeof parsed.value.suggestedTestName === "string" ? parsed.value.suggestedTestName.trim() : "",
+        goalSummary: typeof parsed.value.goalSummary === "string" ? parsed.value.goalSummary.trim() : "",
+        finalState: typeof parsed.value.finalState === "string" ? parsed.value.finalState.trim() : "",
+        importantSteps: parseStringArray(parsed.value.importantSteps),
+        verificationPoints: parseStringArray(parsed.value.verificationPoints),
+        notesForScript: typeof parsed.value.notesForScript === "string" ? parsed.value.notesForScript.trim() : "",
+    };
+    if (!summary.goalSummary || !summary.finalState) {
+        return {
+            kind: "continue",
+            outputItem: buildTextToolOutputItem(callId, "Invalid test_design_complete arguments: goalSummary and finalState are required."),
+        };
+    }
+    session.addToTranscript(`[Design Complete] ${summary.goalSummary}`);
+    const addOutput = context?.addOutput || printCliOutput;
+    addOutput({
+        type: "success",
+        text: "Design exploration complete. Generating final test script...",
+        eventType: "tool_call",
+        runId: context?.runId,
+        stepId: stepContext?.stepId,
+        instructionIndex: stepContext?.instructionIndex,
+        payload: {
+            toolName: "test_design_complete",
+            suggestedTestName: summary.suggestedTestName || null,
+            goalSummary: summary.goalSummary,
+            finalState: summary.finalState,
+        },
+    });
+    return {
+        kind: "continue",
+        outputItem: buildTextToolOutputItem(callId, formatDesignCompletionSummary(summary)),
+        nextTurnConfig: {
+            followUpMessages: [{
+                    role: "user",
+                    content: buildDesignScriptGenerationPrompt(summary),
+                }],
+            toolCapabilitiesOverride: {
+                previewClick: false,
+                assert: false,
+                navigate: false,
+                loadmill: false,
+                testDesignComplete: false,
+            },
+            includeComputerToolOverride: false,
+        },
+    };
+}