@loadmill/droid-cua 2.22.0 → 2.23.0

package/build/index.js

@@ -15,7 +15,7 @@ import { selectDevice } from "./src/cli/device-selector.js";
 import { resolveExecutionAppContext } from "./src/core/app-context.js";
 import { resolveExecutionTestData } from "./src/core/test-data.js";
 import { getDefaultSecretsPath, resolveExecutionSecrets } from "./src/core/secrets.js";
-import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
+import { assertRuntimeCredentialsForProvider, createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
 import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
 import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
@@ -71,6 +71,33 @@ function createCliRuntimeContext(configOverrides = {}) {
         loadmillClient,
     };
 }
+async function assertCliRuntimeCredentials({ llmProviderMode }) {
+    const credentials = resolveRuntimeCredentialsFromEnv(process.env);
+    assertRuntimeCredentialsForProvider({
+        llmProviderMode,
+        credentials,
+    });
+    if (llmProviderMode !== "loadmill") {
+        return;
+    }
+    const loadmillClient = createLoadmillApiClient({
+        token: credentials.loadmillApiToken,
+        baseUrl: credentials.loadmillBaseUrl,
+    });
+    let tokenValidation = null;
+    try {
+        tokenValidation = await loadmillClient.validateApiToken();
+    }
+    catch {
+        throw new Error("Could not validate LOADMILL_API_TOKEN.");
+    }
+    if (tokenValidation?.reason === "invalid") {
+        throw new Error("LOADMILL_API_TOKEN is invalid.");
+    }
+    if (!tokenValidation?.valid) {
+        throw new Error("Could not validate LOADMILL_API_TOKEN.");
+    }
+}
 function createLambdaTestCliBuildName() {
     return `loadmill-droid-cua-cli-${new Date().toISOString().replace(/[:.]/g, "-")}`;
 }
@@ -184,6 +211,9 @@ async function main() {
     let preloadedSecrets = null;
     if (isHeadlessInstructionsRun) {
         preloadedHeadlessConfig = await resolveHeadlessExecutionConfig(args);
+        await assertCliRuntimeCredentials({
+            llmProviderMode: preloadedHeadlessConfig.llmProviderMode,
+        });
         preloadedHeadlessTarget = await resolveHeadlessInstructionsTarget(instructionsFile);
         preloadedSecrets = await resolveExecutionSecrets({
             secretsPath: getDefaultSecretsPath(process.cwd()),

package/build/src/browser/playwright/actions.js

@@ -3,7 +3,8 @@ import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { buildActionEventMeta } from "../../device/action-event-meta.js";
 import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../../device/sensitive-action.js";
-import { getActiveBrowserPage } from "./session-state.js";
+import { normalizeAutomationUrl } from "./url.js";
+import { getActiveBrowserSession, getActiveBrowserPage } from "./session-state.js";
 const KEY_ALIASES = {
     CTRL: "Control",
     CONTROL: "Control",
@@ -58,8 +59,47 @@ function buildMeta(context, action, payload = {}) {
         payload,
     });
 }
+function handleNavigationTimeout(error, description) {
+    if (!(error instanceof Error) || error.name !== "TimeoutError") {
+        throw error;
+    }
+    console.warn(`Page load did not finish within the timeout for ${description}. Continuing with the current page state.`);
+}
+async function gotoWithTimeoutTolerance(page, url) {
+    try {
+        await page.goto(url, { waitUntil: "load" });
+    }
+    catch (error) {
+        handleNavigationTimeout(error, url);
+    }
+}
+async function goBackWithTimeoutTolerance(page) {
+    try {
+        await page.goBack({ waitUntil: "load" });
+    }
+    catch (error) {
+        handleNavigationTimeout(error, "browser history back");
+    }
+}
+async function goForwardWithTimeoutTolerance(page) {
+    try {
+        await page.goForward({ waitUntil: "load" });
+    }
+    catch (error) {
+        handleNavigationTimeout(error, "browser history forward");
+    }
+}
+function updateSessionUrlFromPage(session, page) {
+    const currentUrl = typeof page.url === "function" ? page.url() : null;
+    if (!currentUrl) {
+        return;
+    }
+    session.pageUrls?.set?.(page, currentUrl);
+    session.url = currentUrl;
+}
 export async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const session = getActiveBrowserSession(browserSessionId);
     const page = getActiveBrowserPage(browserSessionId);
     try {
         emitDesktopDebug("browser.action.execute", "device", {
@@ -166,6 +206,30 @@ export async function handleModelAction(browserSessionId, action, scale = 1, con
                 await page.waitForTimeout(timeoutMs);
                 break;
             }
+            case "goto": {
+                const normalizedUrl = normalizeAutomationUrl(action.url);
+                addOutput({
+                    type: "action",
+                    text: `Navigating to ${normalizedUrl}`,
+                    ...buildMeta(context, action, { url: normalizedUrl }),
+                });
+                await gotoWithTimeoutTolerance(page, normalizedUrl);
+                session.pageUrls?.set?.(page, normalizedUrl);
+                session.url = normalizedUrl;
+                break;
+            }
+            case "back": {
+                addOutput({ type: "action", text: "Going back", ...buildMeta(context, action) });
+                await goBackWithTimeoutTolerance(page);
+                updateSessionUrlFromPage(session, page);
+                break;
+            }
+            case "forward": {
+                addOutput({ type: "action", text: "Going forward", ...buildMeta(context, action) });
+                await goForwardWithTimeoutTolerance(page);
+                updateSessionUrlFromPage(session, page);
+                break;
+            }
             default:
                 addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
         }

package/build/src/commands/help.js

@@ -32,7 +32,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --session-mode <mode>  Browser session mode for web runs: persistent or fresh' });
     addOutput({ type: 'info', text: '  --config <file>        Headless execution JSON config for prompt parity' });
     addOutput({ type: 'info', text: '  --llm-provider <mode>  Headless AI provider: loadmill or openai' });
-    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4, loadmill-smart, or gpt-5.4-mini' });
+    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4, loadmill-smart, loadmill-beacon, or gpt-5.4-mini' });
     addOutput({ type: 'info', text: '  --aim-assist <mode>    Headless Aim Assist mode: off, smart, or always' });
     addOutput({ type: 'info', text: '  --context <file>       Optional app context file injected into execution runs' });
     addOutput({ type: 'info', text: '  --no-context           Disable app context for headless execution' });

package/build/src/core/cua-models.js

@@ -2,15 +2,22 @@ export const DEFAULT_OPENAI_CUA_MODEL = "gpt-5.4";
 export const DEFAULT_LOADMILL_CUA_MODEL = "loadmill-smart";
 export const DEFAULT_CUA_MODEL = DEFAULT_LOADMILL_CUA_MODEL;
 export const OPENAI_CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
-export const LOADMILL_CUA_MODELS = [DEFAULT_LOADMILL_CUA_MODEL];
-export const CUA_MODELS = ["gpt-5.4", "loadmill-smart", "gpt-5.4-mini"];
+export const LOADMILL_CUA_MODELS = [DEFAULT_LOADMILL_CUA_MODEL, "loadmill-beacon"];
+export const CUA_MODELS = ["gpt-5.4", "loadmill-smart", "loadmill-beacon", "gpt-5.4-mini"];
 export const CUA_MODEL_OPTIONS = Object.freeze([
     Object.freeze({
         id: "loadmill-smart",
         providerMode: "loadmill",
         providerLabel: "Loadmill",
-        name: "Loadmill Smart",
-        label: "Loadmill Smart (recommended)",
+        name: "Loadmill - Smart",
+        label: "Loadmill - Smart (recommended)",
+    }),
+    Object.freeze({
+        id: "loadmill-beacon",
+        providerMode: "loadmill",
+        providerLabel: "Loadmill",
+        name: "Loadmill - Beacon",
+        label: "Loadmill - Beacon",
     }),
     Object.freeze({
         id: "gpt-5.4",

package/build/src/core/execution-engine.js

@@ -19,6 +19,51 @@ import { handleLoadmillToolCall } from "./tool-handlers/loadmill.js";
 import { handleTestDesignCompleteToolCall } from "./tool-handlers/test-design-complete.js";
 import { handleTypeSensitiveTextToolCall } from "./tool-handlers/type-sensitive-text.js";
 import { buildDebugContext, buildTextToolOutputItem, } from "./tool-handlers/utils.js";
+const HOLO_RESPONSE_ID_PREFIX = "resp_holo_";
+function isHoloResponseId(responseId) {
+    return typeof responseId === "string" && responseId.startsWith(HOLO_RESPONSE_ID_PREFIX);
+}
+function isImageOutputChunk(chunk) {
+    if (!chunk || typeof chunk !== "object") {
+        return false;
+    }
+    if (chunk.type !== "input_image" && chunk.type !== "image_url") {
+        return false;
+    }
+    if (typeof chunk.image_url === "string") {
+        return chunk.image_url.trim().length > 0;
+    }
+    return typeof chunk.image_url?.url === "string" && chunk.image_url.url.trim().length > 0;
+}
+function functionCallOutputHasImage(outputItem) {
+    if (outputItem?.type !== "function_call_output") {
+        return false;
+    }
+    const chunks = [];
+    if (Array.isArray(outputItem.output)) {
+        chunks.push(...outputItem.output);
+    }
+    if (Array.isArray(outputItem.content)) {
+        chunks.push(...outputItem.content);
+    }
+    return chunks.some((chunk) => isImageOutputChunk(chunk));
+}
+function appendImageToFunctionCallOutput(outputItem, screenshotBase64) {
+    if (outputItem?.type !== "function_call_output" || typeof screenshotBase64 !== "string" || !screenshotBase64) {
+        return outputItem;
+    }
+    return {
+        ...outputItem,
+        output: [
+            ...(Array.isArray(outputItem.output) ? outputItem.output : []),
+            {
+                type: "input_image",
+                image_url: `data:image/png;base64,${screenshotBase64}`,
+                detail: "high",
+            },
+        ],
+    };
+}
 function extractActionableOutputItems(items) {
     const entries = [];
     for (const item of items) {
@@ -58,6 +103,129 @@ function collectPendingSafetyChecks(items = []) {
         .filter((item) => item.type === "pending_safety_check")
         .map((item) => ({ id: item.id }));
 }
+const MAX_OBSERVATION_ONLY_TURNS_WITHOUT_ACTION = 10;
+const OBSERVATION_ONLY_RECOVERY_TURN = 5;
+const MAX_RECOVERY_ACTION_SUMMARIES = 8;
+const OBSERVATION_ONLY_RECOVERY_PROMPT = [
+    "You have repeatedly requested screenshots/previews without taking a device action.",
+    "Perform the next real action, choose a meaningfully different action, or stop if this instruction is complete.",
+    "Do not request another screenshot or preview unless you need it after changing the UI.",
+].join(" ");
+function isObservationOnlyActionableItem(actionableItem) {
+    if (actionableItem?.type === "function_call") {
+        return actionableItem.name === "preview_click";
+    }
+    if (actionableItem?.type !== "computer_call") {
+        return false;
+    }
+    return actionableItem.actions.length > 0 &&
+        actionableItem.actions.every((action) => action?.type === "screenshot");
+}
+function parseObservationToolArguments(argumentsText) {
+    if (typeof argumentsText !== "string" || !argumentsText.trim()) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(argumentsText);
+        return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+function buildObservationOnlyPayload(actionableItem) {
+    if (actionableItem?.type === "function_call") {
+        const parsedArguments = parseObservationToolArguments(actionableItem.arguments);
+        return {
+            observationKind: "preview_click",
+            callId: actionableItem.call_id,
+            toolName: actionableItem.name,
+            ...(parsedArguments ? { toolArguments: parsedArguments } : {}),
+        };
+    }
+    return {
+        observationKind: "screenshot_only",
+        callId: actionableItem?.call_id,
+        actionTypes: actionableItem?.actions?.map((action) => action?.type).filter(Boolean) ?? [],
+    };
+}
+function truncateRecoveryText(text, maxLength = 80) {
+    if (typeof text !== "string")
+        return "";
+    return text.length > maxLength ? `${text.slice(0, maxLength - 1)}...` : text;
+}
+function formatActionForRecovery(action) {
+    if (!action || action.type === "screenshot" || action.type === "wait") {
+        return null;
+    }
+    switch (action.type) {
+        case "click":
+            return "Clicked";
+        case "double_click":
+            return "Double-clicked";
+        case "type": {
+            if (action.sensitive) {
+                return `Typed secret${action.sensitiveKey ? `: ${action.sensitiveKey}` : ""}`;
+            }
+            const text = truncateRecoveryText(action.text);
+            return text ? `Typed ${JSON.stringify(text)}` : "Typed text";
+        }
+        case "keypress": {
+            const keys = Array.isArray(action.keys) ? action.keys.join("+") : action.key || action.keys;
+            return keys ? `Pressed ${keys}` : "Pressed a key";
+        }
+        case "scroll":
+            return "Scrolled";
+        default:
+            return null;
+    }
+}
+function buildObservationOnlyRecoveryMessage({ instruction = null, recentRealActions = [] } = {}) {
+    const content = [OBSERVATION_ONLY_RECOVERY_PROMPT];
+    if (typeof instruction === "string" && instruction.trim()) {
+        content.push(`Current instruction: ${instruction.trim()}`);
+    }
+    if (recentRealActions.length > 0) {
+        content.push([
+            "Recent real interactions:",
+            ...recentRealActions.map((actionSummary) => `- ${actionSummary}`),
+        ].join("\n"));
+    }
+    return {
+        role: "user",
+        content: content.join("\n\n"),
+    };
+}
+function getInstructionScrollDirection(instruction) {
+    if (typeof instruction !== "string") {
+        return null;
+    }
+    const normalizedInstruction = instruction.toLowerCase();
+    if (normalizedInstruction.includes("scroll down")) {
+        return 1;
+    }
+    if (normalizedInstruction.includes("scroll up")) {
+        return -1;
+    }
+    return null;
+}
+function normalizeScrollActionForInstruction(action, instruction) {
+    if (action?.type !== "scroll") {
+        return action;
+    }
+    const expectedDirection = getInstructionScrollDirection(instruction);
+    const scrollY = Number(action.scroll_y);
+    if (!expectedDirection || !Number.isFinite(scrollY) || scrollY === 0) {
+        return action;
+    }
+    if (Math.sign(scrollY) === expectedDirection) {
+        return action;
+    }
+    return {
+        ...action,
+        scroll_y: Math.abs(scrollY) * expectedDirection,
+    };
+}
 export class ExecutionEngine {
     constructor(session, options = {}) {
         this.session = session;
@@ -169,9 +337,10 @@ export class ExecutionEngine {
         }
     }
     async requestNextTurn(messages, previousResponseId, context = null, stepContext = null, options = {}) {
+        const continuationMode = options.continuationModeOverride === "fresh" ? "fresh" : "auto";
         return await this.sendCUARequest({
             messages,
-            previousResponseId,
+            previousResponseId: continuationMode === "fresh" ? null : previousResponseId,
             deviceInfo: this.session.deviceInfo,
             debugContext: options.debugContextOverride ?? buildDebugContext(context, stepContext),
             toolCapabilities: options.toolCapabilitiesOverride ?? this.toolCapabilities,
@@ -180,8 +349,21 @@ export class ExecutionEngine {
     }
     async executeActionBatch(actions, currentScreenshot, shouldStop, context = null, stepContext = null, trackAction = null) {
         let sawExplicitScreenshotAction = false;
-        for (const action of actions) {
+        for (const rawAction of actions) {
+            const action = normalizeScrollActionForInstruction(rawAction, context?.currentInstruction);
             const actionEventId = randomUUID();
+            if (action !== rawAction) {
+                emitDesktopDebug("runner.action.normalized", "execution", {
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }, {
+                    reason: "instruction_scroll_direction",
+                    instruction: context?.currentInstruction,
+                    originalAction: rawAction,
+                    normalizedAction: action
+                });
+            }
             if (shouldStop()) {
                 return { stopped: true, sawExplicitScreenshotAction };
             }
@@ -219,6 +401,18 @@ export class ExecutionEngine {
                 shouldStop,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
+                deviceWidth: Number.isFinite(this.session.deviceInfo?.device_width)
+                    ? Math.round(this.session.deviceInfo.device_width)
+                    : undefined,
+                deviceHeight: Number.isFinite(this.session.deviceInfo?.device_height)
+                    ? Math.round(this.session.deviceInfo.device_height)
+                    : undefined,
+                fallbackScrollStartX: Number.isFinite(this.session.deviceInfo?.device_width)
+                    ? Math.round(this.session.deviceInfo.device_width / 2)
+                    : undefined,
+                fallbackScrollStartY: Number.isFinite(this.session.deviceInfo?.device_height)
+                    ? Math.round(this.session.deviceInfo.device_height / 2)
+                    : undefined,
                 eventId: actionEventId,
                 captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
                 ...(annotatedScreenshot?.screenshotPath ? { screenshotPath: annotatedScreenshot.screenshotPath } : {}),
@@ -366,7 +560,8 @@ export class ExecutionEngine {
         const addOutput = context?.addOutput || printCliOutput;
         let newResponseId = response.id;
         let currentScreenshot = options.currentScreenshot ?? null;
-        const shouldStop = () => Boolean(trackAction?.());
+        const originalTrackAction = trackAction;
+        const shouldStop = () => Boolean(originalTrackAction?.());
         const summary = {
             responseId: newResponseId,
             invokedFunctionTools: [],
@@ -383,6 +578,93 @@ export class ExecutionEngine {
             ...extra
         });
         let refreshBeforeNextComputerAction = false;
+        let observationOnlyTurnsSinceLastAction = 0;
+        let observationOnlyRecoverySent = false;
+        let pendingObservationOnlyRecoveryMessage = null;
+        const recentRealActions = [];
+        const recordRecentRealAction = (action) => {
+            const summaryText = formatActionForRecovery(action);
+            if (!summaryText)
+                return;
+            recentRealActions.push(summaryText);
+            if (recentRealActions.length > MAX_RECOVERY_ACTION_SUMMARIES) {
+                recentRealActions.shift();
+            }
+        };
+        const recordRecentRealActions = (actions = []) => {
+            for (const action of actions)
+                recordRecentRealAction(action);
+        };
+        const wrappedTrackAction = (action = null) => {
+            if (action)
+                recordRecentRealAction(action);
+            return originalTrackAction?.(action);
+        };
+        const resetObservationOnlyStreak = () => {
+            observationOnlyTurnsSinceLastAction = 0;
+            observationOnlyRecoverySent = false;
+            pendingObservationOnlyRecoveryMessage = null;
+        };
+        const buildObservationLoopPayload = (actionableItem) => ({
+            reason: "observation_only_loop",
+            responseId: newResponseId,
+            observationOnlyTurnsSinceLastAction,
+            maxObservationOnlyTurnsWithoutAction: MAX_OBSERVATION_ONLY_TURNS_WITHOUT_ACTION,
+            recoveryTurn: OBSERVATION_ONLY_RECOVERY_TURN,
+            recoverySent: observationOnlyRecoverySent,
+            ...buildObservationOnlyPayload(actionableItem),
+        });
+        const queueObservationOnlyRecovery = (actionableItem) => {
+            if (observationOnlyRecoverySent ||
+                observationOnlyTurnsSinceLastAction < OBSERVATION_ONLY_RECOVERY_TURN) {
+                return false;
+            }
+            observationOnlyRecoverySent = true;
+            pendingObservationOnlyRecoveryMessage = buildObservationOnlyRecoveryMessage({
+                instruction: stepContext?.instruction,
+                recentRealActions,
+            });
+            addOutput({
+                type: "warning",
+                text: "The agent seems confused. Trying to get it back on track.",
+                eventType: "system_message",
+                ...eventMeta({
+                    payload: {
+                        ...buildObservationLoopPayload(actionableItem),
+                        recoveryPrompt: pendingObservationOnlyRecoveryMessage.content,
+                        ...(typeof stepContext?.instruction === "string" && stepContext.instruction.trim()
+                            ? { instruction: stepContext.instruction.trim() }
+                            : {}),
+                        ...(recentRealActions.length > 0 ? { recentRealActions: [...recentRealActions] } : {}),
+                    },
+                }),
+            });
+            return true;
+        };
+        const failObservationOnlyLoop = (actionableItem) => {
+            const error = "The agent got too confused. Stopping the test.";
+            summary.failed = true;
+            summary.error = error;
+            summary.observationLoopDetected = true;
+            addOutput({
+                type: "error",
+                text: error,
+                eventType: "error",
+                ...eventMeta({
+                    payload: buildObservationLoopPayload(actionableItem),
+                }),
+            });
+            newResponseId = null;
+        };
+        const updateObservationOnlyStreak = (actionableItem) => {
+            observationOnlyTurnsSinceLastAction += 1;
+            if (observationOnlyTurnsSinceLastAction >= MAX_OBSERVATION_ONLY_TURNS_WITHOUT_ACTION) {
+                failObservationOnlyLoop(actionableItem);
+                return true;
+            }
+            queueObservationOnlyRecovery(actionableItem);
+            return false;
+        };
         while (true) {
             // Check for interruption before processing next batch of actions
             if (shouldStop()) {
@@ -409,6 +691,10 @@ export class ExecutionEngine {
             if (!actionableItem?.call_id) {
                 break;
             }
+            const observationOnlyTurn = isObservationOnlyActionableItem(actionableItem);
+            if (!observationOnlyTurn) {
+                resetObservationOnlyStreak();
+            }
             if (actionableItem.type === "function_call") {
                 const toolResult = await this.handleFunctionToolCall(actionableItem.name, actionableItem.call_id, actionableItem.arguments, context, stepContext);
                 summary.invokedFunctionTools.push(actionableItem.name);
@@ -445,15 +731,51 @@ export class ExecutionEngine {
                     newResponseId = null;
                     break;
                 }
-                const nextTurnMessages = [toolResult.outputItem];
+                if (Array.isArray(toolResult.executedActions)) {
+                    recordRecentRealActions(toolResult.executedActions);
+                }
+                if (observationOnlyTurn && updateObservationOnlyStreak(actionableItem)) {
+                    break;
+                }
                 const nextTurnConfig = toolResult.nextTurnConfig ?? {};
+                let nextTurnOutputItem = toolResult.outputItem;
+                if (nextTurnConfig.attachFreshScreenshotOnContinuation === true
+                    && isHoloResponseId(newResponseId)
+                    && nextTurnOutputItem?.type === "function_call_output"
+                    && !functionCallOutputHasImage(nextTurnOutputItem)) {
+                    let continuationScreenshot = toolResult.currentScreenshot ?? null;
+                    let screenshotBase64 = typeof continuationScreenshot?.base64 === "string" && continuationScreenshot.base64
+                        ? continuationScreenshot.base64
+                        : null;
+                    if (!screenshotBase64) {
+                        screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+                        continuationScreenshot = await this.recordScreenshot(screenshotBase64, {
+                            runId: context?.runId,
+                            sessionId: context?.sessionId,
+                            stepId: stepContext?.stepId,
+                            instructionIndex: stepContext?.instructionIndex,
+                            captureSource: CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
+                        });
+                    }
+                    nextTurnOutputItem = appendImageToFunctionCallOutput(nextTurnOutputItem, screenshotBase64);
+                    currentScreenshot = continuationScreenshot ?? currentScreenshot;
+                }
+                const startsFreshResponseChain = nextTurnConfig.continuationMode === "fresh";
+                // Fresh follow-ups start a new provider response chain. Tool outputs from
+                // the previous chain cannot be acknowledged without the matching tool call.
+                const nextTurnMessages = startsFreshResponseChain ? [] : [nextTurnOutputItem];
                 if (Array.isArray(nextTurnConfig.followUpMessages) && nextTurnConfig.followUpMessages.length > 0) {
                     nextTurnMessages.push(...nextTurnConfig.followUpMessages);
                 }
+                if (pendingObservationOnlyRecoveryMessage) {
+                    nextTurnMessages.push(pendingObservationOnlyRecoveryMessage);
+                    pendingObservationOnlyRecoveryMessage = null;
+                }
                 response = await this.requestNextTurn(nextTurnMessages, newResponseId, context, stepContext, {
                     debugContextOverride: options.debugContextOverride,
                     toolCapabilitiesOverride: nextTurnConfig.toolCapabilitiesOverride,
                     includeComputerToolOverride: nextTurnConfig.includeComputerToolOverride,
+                    continuationModeOverride: nextTurnConfig.continuationMode,
                 });
                 newResponseId = response.id;
                 if (toolResult.kind === "close_turn") {
@@ -470,7 +792,7 @@ export class ExecutionEngine {
                 refreshBeforeNextComputerAction = false;
                 continue;
             }
-            const actionBatchResult = await this.executeActionBatch(actions, currentScreenshot, shouldStop, context, stepContext, trackAction);
+            const actionBatchResult = await this.executeActionBatch(actions, currentScreenshot, shouldStop, context, stepContext, wrappedTrackAction);
             if (actionBatchResult.stopped) {
                 summary.stopped = true;
                 return finalize();
@@ -496,6 +818,13 @@ export class ExecutionEngine {
                 refreshBeforeNextComputerAction = true;
             }
             currentScreenshot = followUp.currentScreenshot ?? currentScreenshot;
+            if (observationOnlyTurn && updateObservationOnlyStreak(actionableItem)) {
+                break;
+            }
+            if (pendingObservationOnlyRecoveryMessage) {
+                followUp.input.push(pendingObservationOnlyRecoveryMessage);
+                pendingObservationOnlyRecoveryMessage = null;
+            }
             response = await this.requestNextTurn(followUp.input, newResponseId, context, stepContext);
             newResponseId = response.id;
         }

package/build/src/core/prompts.js

@@ -1,3 +1,3 @@
 export { buildBaseSystemPrompt } from "../prompts/base.js";
-export { buildDesignModePrompt, buildDesignRecoveryPrompt, buildDesignScriptGenerationPrompt, } from "../prompts/design.js";
+export { buildDesignModePrompt, buildDesignRecoveryPrompt, buildDesignScriptFormatRetryPrompt, buildDesignScriptGenerationPrompt, } from "../prompts/design.js";
 export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, buildSecretKeysContextSection, buildTestDataContextSection, } from "../prompts/execution.js";

package/build/src/core/report/render-html-report.js

@@ -2,7 +2,7 @@
 // HTML document that reproduces the Desktop App's run-details view. Pure: no I/O and
 // no external references — screenshots must already be inlined as data URLs in
 // `event.payload.screenshotDataUrl`. Use generate-html-report.js to inline them.
-import { assertionData, formatDuration, hasNearbyAssertionCard, isAssistantAssertionSummary, isDuplicateAssertionLine, isHiddenExecutionLogEvent, runSummaryData, } from "./run-log-model.js";
+import { assertionData, formatDuration, hasNearbyAssertionCard, isAssistantAssertionSummary, isDuplicateAssertionLine, isHiddenExecutionLogEvent, runSummaryData, stepCompletionData, stepCompletionReasonTooltip, } from "./run-log-model.js";
 // ---------- text / value formatting (mirrors the Desktop App) ----------
 function escapeHtml(value) {
     return String(value ?? "")
@@ -18,7 +18,7 @@ function formatRunTimestamp(iso) {
     const parsed = Date.parse(iso);
     if (!Number.isFinite(parsed))
         return "—";
-    return new Date(parsed).toLocaleString();
+    return new Date(parsed).toLocaleString("en-US");
 }
 function formatRunDuration(durationSeconds) {
     if (durationSeconds === null || durationSeconds === undefined || !Number.isFinite(durationSeconds)) {
@@ -121,6 +121,11 @@ function renderEvent(event, shots) {
     }
     const isAssistant = event.eventType === "assistant_message" || event.kind === "assistant";
     if (isAssistant) {
+        const stepCompletion = stepCompletionData(event);
+        if (stepCompletion) {
+            const tooltip = stepCompletionReasonTooltip(stepCompletion);
+            return `<div class="dc-line"${tooltip ? ` title="${escapeHtml(tooltip)}"` : ""}><span aria-hidden="true">&#9873;</span><span>Step complete</span></div>`;
+        }
         return `<div class="dc-assistant">${escapeHtml(event.text)}</div>`;
     }
     const assertion = assertionData(event);
@@ -239,7 +244,7 @@ body { margin: 0; background: #f1f5f9; color: #0f172a; font-family: -apple-syste
 .dc-row-right { justify-content: flex-end; }
 .dc-bubble { display: inline-block; max-width: 70%; background: #f1f5f9; border: 1px solid #e2e8f0; border-radius: 16px; border-bottom-right-radius: 6px; padding: 8px 14px; color: #0f172a; }
 .dc-assistant { max-width: 82%; color: #334155; }
-.dc-line { display: flex; align-items: center; gap: 8px; color: #475569; font-size: 13px; }
+.dc-line { display: flex; align-items: center; gap: 8px; color: #475569; }
 .dc-retry { color: #b45309; }
 .dc-error { color: #b91c1c; }
 .dc-warning { color: #b45309; }