libre-webui 0.27.0 → 0.28.0

package/backend/dist/services/workAgentService.js

@@ -16,8 +16,11 @@
  */
 import workModelProviderService, { WORK_TOOL_ARGUMENTS_ERROR_METADATA_KEY, WORK_TOOL_ARGUMENTS_ERROR_MESSAGE, } from './workModelProviderService.js';
 import workEventService from './workEventService.js';
-import { buildWorkAgentSystemPrompt, buildWorkBudgetExhaustionPrompt, buildWorkEmptyRoundNudgePrompt, WORK_AGENT_SKILLS, WORK_WRITE_FILE_RECOMMENDED_CHARS, workToolCallBudget, } from './workAgentGuidance.js';
-import workRuntimeService from './workRuntimeService.js';
+import { buildWorkAgentSystemPrompt, buildWorkBudgetExhaustionPrompt, buildWorkComputerAmbiguityPrompt, buildWorkComputerStallPrompt, buildWorkEmptyRoundNudgePrompt, workAgentSkillsForContext, WORK_WRITE_FILE_RECOMMENDED_CHARS, workToolCallBudget, } from './workAgentGuidance.js';
+import workPolicyService from './workPolicyService.js';
+import workRuntimeService, { WORK_COMPUTER_ACTION_LIMIT, } from './workRuntimeService.js';
+import workScreenControlService, { WORK_SCREEN_ASSIST_TIMEOUT_MS, } from './workScreenControlService.js';
+import workComputerTeachService from './workComputerTeachService.js';
 import { isWebSearchAvailable, userCanUseWebSearch, webSearch, } from './webSearchService.js';
 import { userModel } from '../models/userModel.js';
 import workTaskService, { WORK_MESSAGE_METADATA_MAX_BYTES, WorkConflictError, WorkNotFoundError, } from './workTaskService.js';
@@ -82,12 +85,106 @@ export const WORK_WEB_SEARCH_TOOL_SCHEMA = functionTool('web_search', 'Search th
         description: "Optional result count. Defaults to 5, capped by the server's configured limit.",
     },
 }, ['query']);
+// Offered only when the task's policy enables the Work Computer and the
+// task has network access: the agent's eyes and hands on the GUI session.
+export const WORK_COMPUTER_TOOL_SCHEMAS = [
+    functionTool('computer_observe', "Look at the task's virtual computer screen. Returns the current screenshot with the cursor position, active window, page URL, and which element holds keyboard focus.", {}),
+    functionTool('computer_act', 'Perform batched mouse and keyboard actions on the virtual computer, then return the screenshot after they settle. The batch stops early (and tells you) if the window, title, or window count changes mid-batch — later coordinates would target the previous screen — or if a "focus" assertion fails.', {
+        actions: {
+            type: 'array',
+            description: `Up to ${WORK_COMPUTER_ACTION_LIMIT} actions executed in order. Each action is an object with "type" plus type-specific fields: ` +
+                'move {x,y}; click / double_click / right_click {x,y optional — clicks the current position without them}; ' +
+                'type {text}; key {keys, an xdotool name or chord such as "Return" or "ctrl+l"}; ' +
+                'scroll {direction: "up"|"down", amount optional, x,y optional}; ' +
+                'scroll_until {direction, target: {text} or {edge: "top"|"bottom"}, maxAmount optional ≤30, x,y optional} — goal-directed scrolling that stops when the target text is visible and reports a visibility receipt, always preferable to blind scroll amounts; ' +
+                'wait {ms, at most 5000}. ' +
+                'type and key accept an optional "focus" string: the action runs only when the focused element, page URL, or window title contains it — use it before typing anything that must land in a specific field. ' +
+                'Clicks at explicit coordinates return a receipt saying whether pixels near the click changed.',
+            items: {
+                type: 'object',
+                properties: {
+                    type: {
+                        type: 'string',
+                        enum: [
+                            'move',
+                            'click',
+                            'double_click',
+                            'right_click',
+                            'type',
+                            'key',
+                            'scroll',
+                            'scroll_until',
+                            'wait',
+                        ],
+                    },
+                    x: { type: 'integer' },
+                    y: { type: 'integer' },
+                    text: { type: 'string' },
+                    keys: { type: 'string' },
+                    focus: { type: 'string' },
+                    direction: { type: 'string', enum: ['up', 'down'] },
+                    amount: { type: 'integer' },
+                    maxAmount: { type: 'integer' },
+                    target: {
+                        type: 'object',
+                        properties: {
+                            text: { type: 'string' },
+                            edge: { type: 'string', enum: ['top', 'bottom'] },
+                        },
+                    },
+                    ms: { type: 'integer' },
+                },
+                required: ['type'],
+            },
+        },
+        subgoal: {
+            type: 'string',
+            description: 'One short sentence naming what this batch is meant to achieve (e.g. "open the settings page"). Persisted with the result as a checkpoint, and echoed back if the loop needs to recover — declare it on every consequential batch.',
+        },
+        expect: {
+            type: 'object',
+            description: 'Optional expected outcome, verified by the runtime after the batch settles (polled up to withinMs, default 5000). ' +
+                'Declare it for any consequential batch so you learn whether the intent succeeded, not merely that input was delivered. ' +
+                '"pending" in the result means not observed before the deadline — re-observe before assuming failure.',
+            properties: {
+                titleContains: {
+                    type: 'string',
+                    description: 'The active window title should contain this text.',
+                },
+                urlContains: {
+                    type: 'string',
+                    description: 'The browser URL should contain this text.',
+                },
+                regionChanged: {
+                    type: 'object',
+                    description: 'Pixels in this screen region should change relative to the start of the batch.',
+                    properties: {
+                        x: { type: 'integer' },
+                        y: { type: 'integer' },
+                        width: { type: 'integer' },
+                        height: { type: 'integer' },
+                    },
+                    required: ['x', 'y', 'width', 'height'],
+                },
+                withinMs: { type: 'integer' },
+            },
+        },
+    }, ['actions']),
+    functionTool('request_takeover', 'Ask the user to take control of the virtual computer screen — for signing in, a CAPTCHA, 2FA, or any step you must not perform yourself. Waits until the user finishes and hands control back (or a timeout passes). While the user is driving, your computer tools are blocked.', {
+        reason: stringProperty('What you need the user to do on the screen, stated concretely.'),
+    }, ['reason']),
+];
 export async function workToolSchemasForTask(task) {
-    return task.networkEnabled &&
+    const schemas = [...WORK_TOOL_SCHEMAS];
+    if (task.networkEnabled &&
         (await isWebSearchAvailable()) &&
-        (await userCanUseWebSearch(await userModel.getUserById(task.userId)))
-        ? [...WORK_TOOL_SCHEMAS, WORK_WEB_SEARCH_TOOL_SCHEMA]
-        : WORK_TOOL_SCHEMAS;
+        (await userCanUseWebSearch(await userModel.getUserById(task.userId)))) {
+        schemas.push(WORK_WEB_SEARCH_TOOL_SCHEMA);
+    }
+    if (await workRuntimeService.computerToolsAvailable(task)) {
+        schemas.push(...WORK_COMPUTER_TOOL_SCHEMAS);
+    }
+    return schemas;
 }
 export class WorkAgentService {
     constructor() {
@@ -437,13 +534,31 @@ export class WorkAgentService {
                 status: 'preparing',
                 phase: 'preparing',
             }, 'transition:preparing', durableAttemptIdentity);
-            for (const skill of WORK_AGENT_SKILLS) {
+            const computerAvailable = await workRuntimeService.computerToolsAvailable(task);
+            let taughtSkills = [];
+            if (computerAvailable) {
+                try {
+                    taughtSkills =
+                        await workComputerTeachService.taughtSkillsForUser(userId);
+                }
+                catch (error) {
+                    logger.warn(`Could not load taught skills for run ${runId}:`, error);
+                }
+            }
+            for (const skill of workAgentSkillsForContext({ computerAvailable })) {
                 await workEventService.publish(taskId, runId, 'skill_loaded', {
                     id: skill.id,
                     name: skill.title,
                     description: skill.instructions[0],
                 }, `skill:${skill.id}`, durableAttemptIdentity);
             }
+            for (const skill of taughtSkills) {
+                await workEventService.publish(taskId, runId, 'skill_loaded', {
+                    id: `taught:${skill.slug}`,
+                    name: skill.name,
+                    description: 'Taught procedure demonstrated by the user.',
+                }, `skill:taught:${skill.slug}`, durableAttemptIdentity);
+            }
             releaseExecutionLease = await workRuntimeService.prepare(task, controller.signal);
             await this.throwIfCancelled(runId, controller);
             await workTaskService.updateRun(runId, 'running', { started: true });
@@ -456,7 +571,7 @@ export class WorkAgentService {
             };
             const providerRoutingFingerprint = await workModelProviderService.getRoutingFingerprint(run.model, providerSelection, userId);
             const providerStateScope = await workModelProviderService.getResponsesStateScope(run.model, providerSelection, userId);
-            const messages = await this.contextMessages(task, roundLimit, providerStateScope, run);
+            const messages = await this.contextMessages(task, roundLimit, computerAvailable, taughtSkills, providerStateScope, run);
             let totalToolCalls = 0;
             let accumulatedInputTokens = 0;
             let accumulatedOutputTokens = 0;
@@ -464,13 +579,53 @@ export class WorkAgentService {
             let streamedReasoningTotal = '';
             let budgetReason = 'round';
             let emptyRoundNudges = 0;
+            // Computer-loop stall detection: an identical action against a screen
+            // whose pixels did not change is the signature of a grounding loop —
+            // a dead coordinate, a click that never lands. One recovery notice,
+            // then a handoff, instead of burning the remaining rounds.
+            let computerStallSignature = '';
+            let computerStallRepeats = 0;
+            let computerStallNudged = false;
+            // Ambiguity budget: consecutive unverified expectations mean the loop
+            // is acting on assumptions; one re-grounding notice before it
+            // compounds.
+            let consecutivePendingExpectations = 0;
+            let ambiguityNudged = false;
+            const computerContext = {};
+            const loopStats = {
+                rounds: 0,
+                toolCalls: 0,
+                screenshots: 0,
+                fences: 0,
+                expectationsPassed: 0,
+                expectationsPending: 0,
+                stallNudges: 0,
+                ambiguityNudges: 0,
+            };
             // Reasoning-channel models (DeepSeek and friends) sometimes place
             // their entire answer in reasoning and leave content empty. Keep the
             // last reasoning so a run can end with the model's actual findings
             // instead of a placeholder.
             let lastReasoningContent = '';
+            // Mid-run user messages join the model context at round boundaries,
+            // so the user can steer the agent without cancelling the run. Seed
+            // with what the restored context already contains so nothing repeats.
+            const injectedMidRunMessages = new Set((await workTaskService.getMessages(taskId))
+                .filter(message => message.metadata?.midRun === true)
+                .map(message => message.id));
             roundLoop: for (let round = 0; round < roundLimit; round++) {
                 await this.throwIfCancelled(runId, controller);
+                loopStats.rounds = round + 1;
+                for (const message of await workTaskService.getMessages(taskId)) {
+                    if (message.runId === runId &&
+                        message.role === 'user' &&
+                        message.kind === 'message' &&
+                        message.metadata?.midRun === true &&
+                        !injectedMidRunMessages.has(message.id)) {
+                        injectedMidRunMessages.add(message.id);
+                        messages.push({ role: 'user', content: message.content });
+                    }
+                }
                 await workEventService.publish(taskId, runId, 'run_state', {
                     status: 'running',
                     phase: 'thinking',
@@ -614,6 +769,7 @@ export class WorkAgentService {
                     await workTaskService.updateTaskStatus(taskId, 'completed');
                     await workEventService.publish(taskId, runId, 'done', {
                         status: 'completed',
+                        loopStats,
                     }, 'terminal:completed');
                     return;
                 }
@@ -644,7 +800,10 @@ export class WorkAgentService {
                 }
                 for (const call of toolCalls) {
                     await this.throwIfCancelled(runId, controller);
-                    const toolCallMetadata = summarizeToolCall(call);
+                    const toolCallMetadata = {
+                        ...summarizeToolCall(call),
+                        round: round + 1,
+                    };
                     const toolCallMessage = await workTaskService.addMessage(taskId, runId, 'assistant', 'tool_call', `Calling ${call.function.name}`, toolCallMetadata);
                     await workEventService.publish(taskId, runId, 'run_state', {
                         status: 'running',
@@ -661,11 +820,13 @@ export class WorkAgentService {
                         message: toolCallMessage,
                     }, `message:${toolCallMessage.id}`);
                     let toolOutput;
+                    let toolImages;
                     let toolMetadata = {
                         name: call.function.name,
                         toolCallId: call.id,
                         toolName: call.function.name,
                     };
+                    const toolStartedAt = Date.now();
                     try {
                         const validationError = toolValidationErrors.get(call);
                         if (validationError) {
@@ -674,8 +835,12 @@ export class WorkAgentService {
                         if (hasInvalidToolArguments) {
                             throw new WorkAgentHttpError('This tool call was not executed because another call in the same model response had incomplete or invalid arguments. Retry the batch with smaller payloads.', 400, 'WORK_TOOL_BATCH_NOT_EXECUTED');
                         }
-                        const result = await this.executeTool(task, call);
+                        const result = await this.executeTool(task, call, {
+                            signal: controller.signal,
+                            computer: computerContext,
+                        });
                         toolOutput = result.content;
+                        toolImages = result.images;
                         toolMetadata = { ...toolMetadata, ...result.metadata };
                     }
                     catch (error) {
@@ -685,6 +850,25 @@ export class WorkAgentService {
                                 : 'Tool error.';
                         toolMetadata.error = true;
                     }
+                    toolMetadata.durationMs = Date.now() - toolStartedAt;
+                    loopStats.toolCalls += 1;
+                    if (toolMetadata.screenshot === true)
+                        loopStats.screenshots += 1;
+                    if (toolMetadata.fence)
+                        loopStats.fences += 1;
+                    const expectOutcome = toolMetadata.expect?.outcome;
+                    if (expectOutcome === 'passed')
+                        loopStats.expectationsPassed += 1;
+                    if (expectOutcome === 'pending')
+                        loopStats.expectationsPending += 1;
+                    if (call.function.name === 'computer_act') {
+                        if (expectOutcome === 'pending') {
+                            consecutivePendingExpectations += 1;
+                        }
+                        else if (expectOutcome === 'passed') {
+                            consecutivePendingExpectations = 0;
+                        }
+                    }
                     const boundedOutput = boundPersistedToolOutput(toolOutput);
                     if (boundedOutput !== toolOutput) {
                         toolMetadata.outputTruncated = true;
@@ -704,6 +888,59 @@ export class WorkAgentService {
                         content: toolOutput,
                         tool_name: call.function.name,
                         tool_call_id: call.id,
+                        ...(toolImages?.length ? { images: toolImages } : {}),
+                    });
+                    // Screenshots are live-context-only: persisted messages keep the
+                    // text observation, and only the most recent screenshots stay in
+                    // model context so a long computer session cannot flood tokens.
+                    retainRecentWorkImages(messages);
+                    if ((call.function.name === 'computer_act' ||
+                        call.function.name === 'computer_observe') &&
+                        typeof toolMetadata.screenshotSha256 === 'string') {
+                        let argsSignature = '';
+                        try {
+                            argsSignature = JSON.stringify(call.function.arguments ?? '').slice(0, 2000);
+                        }
+                        catch {
+                            argsSignature = '';
+                        }
+                        const signature = [
+                            toolMetadata.screenshotSha256,
+                            call.function.name,
+                            argsSignature,
+                        ].join('|');
+                        if (signature === computerStallSignature) {
+                            computerStallRepeats += 1;
+                        }
+                        else {
+                            computerStallSignature = signature;
+                            computerStallRepeats = 0;
+                        }
+                    }
+                }
+                if (computerStallRepeats >= 2) {
+                    computerStallRepeats = 0;
+                    computerStallSignature = '';
+                    if (!computerStallNudged) {
+                        computerStallNudged = true;
+                        loopStats.stallNudges += 1;
+                        messages.push({
+                            role: 'user',
+                            content: buildWorkComputerStallPrompt(computerContext.lastSubgoal),
+                        });
+                    }
+                    else {
+                        budgetReason = 'computer-stall';
+                        break roundLoop;
+                    }
+                }
+                if (consecutivePendingExpectations >= 2 && !ambiguityNudged) {
+                    ambiguityNudged = true;
+                    consecutivePendingExpectations = 0;
+                    loopStats.ambiguityNudges += 1;
+                    messages.push({
+                        role: 'user',
+                        content: buildWorkComputerAmbiguityPrompt(computerContext.lastSubgoal),
                     });
                 }
             }
@@ -730,7 +967,9 @@ export class WorkAgentService {
                                 ...messages,
                                 {
                                     role: 'user',
-                                    content: `${buildWorkBudgetExhaustionPrompt()}\nThe ${budgetReason} budget was reached.`,
+                                    content: `${buildWorkBudgetExhaustionPrompt()}\n${budgetReason === 'computer-stall'
+                                        ? 'The run was stopped because repeated computer actions kept producing an unchanged screen even after a recovery notice.'
+                                        : `The ${budgetReason} budget was reached.`}`,
                                 },
                             ],
                             tools: [],
@@ -789,6 +1028,7 @@ export class WorkAgentService {
             await workTaskService.addMessage(taskId, runId, 'assistant', 'message', handoffContent, {
                 budgetHandoff: true,
                 budgetReason,
+                loopStats,
                 ...handoffProviderStateMetadata,
             });
             await settleExecutionContainer();
@@ -800,6 +1040,7 @@ export class WorkAgentService {
             await workEventService.publish(taskId, runId, 'done', {
                 status: 'needs_input',
                 budgetReason,
+                loopStats,
             }, 'terminal:needs_input');
         }
         catch (error) {
@@ -877,6 +1118,10 @@ export class WorkAgentService {
         }
     }
     async cleanupExecutionContainer(task) {
+        // A watched Work Computer screen outlives the run for the same reason a
+        // preview does: someone is looking at it.
+        if (workRuntimeService.screenSessionCount(task.id) > 0)
+            return;
         try {
             // A preview is intentionally allowed to outlive the model run so the
             // user can inspect it in the Work pane. Preserve only a verified ready
@@ -909,13 +1154,15 @@ export class WorkAgentService {
             throw new WorkAgentHttpError('The provider routing changed while this Work run was active. Start a new run with the updated provider configuration.', 409, 'WORK_PROVIDER_ROUTING_CHANGED');
         }
     }
-    async contextMessages(task, roundLimit, providerStateScope, provider = task) {
+    async contextMessages(task, roundLimit, computerAvailable, taughtSkills, providerStateScope, provider = task) {
         const persisted = restorePersistedWorkContext(await workTaskService.getRecentModelContextMessages(task.id, 30), provider, providerStateScope);
         return [
             {
                 role: 'system',
                 content: buildWorkAgentSystemPrompt({
                     networkEnabled: task.networkEnabled,
+                    computerAvailable,
+                    ...(taughtSkills.length > 0 ? { taughtSkills } : {}),
                     previewPort: workRuntimeService.previewPort,
                     roundBudget: roundLimit,
                     commandTimeoutMs: workRuntimeService.limits.commandTimeoutMs,
@@ -925,7 +1172,7 @@ export class WorkAgentService {
             ...persisted,
         ];
     }
-    async executeTool(task, call) {
+    async executeTool(task, call, context = {}) {
         const args = call.function.arguments;
         switch (call.function.name) {
             case 'list_files': {
@@ -1005,10 +1252,78 @@ export class WorkAgentService {
                     onStopped: () => workTaskService.updatePreview(task.id, 'stopped'),
                 });
                 return { content: 'Preview stopped.' };
+            case 'computer_observe': {
+                await this.assertComputerToolsAvailable(task);
+                await this.assertComputerNotHumanControlled(task);
+                return computerToolResult(await workRuntimeService.computerObserve(task), undefined, context.computer);
+            }
+            case 'computer_act': {
+                await this.assertComputerToolsAvailable(task);
+                await this.assertComputerNotHumanControlled(task);
+                const actions = Array.isArray(args.actions) ? args.actions : [];
+                const subgoal = optionalString(args.subgoal)?.trim().slice(0, 300);
+                const observation = await workRuntimeService.computerAct(task, actions, args.expect);
+                const result = computerToolResult(observation, actions.length, context.computer);
+                if (subgoal) {
+                    // The checkpoint rides the durable transcript: goal, actions, and
+                    // verified outcome persist together for resume and audit.
+                    result.metadata.subgoal = subgoal;
+                    result.content = `Subgoal: ${subgoal}\n${result.content}`;
+                    if (context.computer)
+                        context.computer.lastSubgoal = subgoal;
+                }
+                return result;
+            }
+            case 'request_takeover': {
+                await this.assertComputerToolsAvailable(task);
+                if ((await workPolicyService.resolve(task.policyId)).takeoverEnabled ===
+                    false) {
+                    return {
+                        content: "This task's Work policy does not allow human takeover of the screen. No one can be handed control; state the exact blocker in your response so the user can address it another way.",
+                        metadata: { outcome: 'disabled' },
+                    };
+                }
+                const reason = requiredString(args.reason, 'reason').slice(0, 1000);
+                const outcome = await workScreenControlService.waitForAssist(task.id, reason, {
+                    timeoutMs: WORK_SCREEN_ASSIST_TIMEOUT_MS,
+                    signal: context.signal,
+                });
+                if (outcome === 'released') {
+                    return {
+                        content: 'The user took control of the computer and has handed it back. Use computer_observe to see the current screen before continuing.',
+                        metadata: { outcome },
+                    };
+                }
+                if (outcome === 'still_controlled') {
+                    return {
+                        content: 'The user is still controlling the computer. Your computer tools stay blocked until they finish; continue with other work or wait.',
+                        metadata: { outcome },
+                    };
+                }
+                return {
+                    content: 'No one took over the screen within the waiting period. State the exact blocker in your response so the user can handle it later.',
+                    metadata: { outcome },
+                };
+            }
             default:
                 throw new WorkAgentHttpError(`Unknown tool: ${call.function.name}`, 400, 'WORK_UNKNOWN_TOOL');
         }
     }
+    async assertComputerToolsAvailable(task) {
+        if (!(await workRuntimeService.computerToolsAvailable(task))) {
+            throw new WorkAgentHttpError('The Work Computer is not available for this task.', 403, 'WORK_COMPUTER_UNAVAILABLE');
+        }
+    }
+    /**
+     * While a human holds the control lease, the agent's eyes and hands are
+     * both off: acting would fight the user's input, and observing could
+     * capture usernames or one-time codes typed during a sign-in.
+     */
+    async assertComputerNotHumanControlled(task) {
+        if (await workScreenControlService.current(task.id)) {
+            throw new WorkAgentHttpError('A user is controlling the Work Computer right now. Wait for them to finish (request_takeover reports when control is handed back).', 409, 'WORK_COMPUTER_HUMAN_CONTROL');
+        }
+    }
     async throwIfCancelled(runId, controller) {
         if (controller.signal.aborted ||
             (await workTaskService.getRun(runId))?.status === 'cancelled') {
@@ -1510,7 +1825,12 @@ function validateToolCallArguments(call) {
         case 'list_files':
         case 'start_preview':
         case 'stop_preview':
+        case 'computer_observe':
             return undefined;
+        case 'computer_act':
+            return Array.isArray(args.actions) && args.actions.length > 0
+                ? undefined
+                : `The model returned invalid arguments for computer_act: "actions" must be a non-empty array of action objects.`;
         case 'read_file':
         case 'delete_file':
             return invalidRequiredString('path');
@@ -1521,6 +1841,8 @@ function validateToolCallArguments(call) {
         case 'search_files':
         case 'web_search':
             return invalidRequiredString('query');
+        case 'request_takeover':
+            return invalidRequiredString('reason');
         case 'run_command':
             return invalidRequiredString('command');
         default:
@@ -1579,11 +1901,179 @@ function summarizeToolCall(call) {
         case 'start_preview':
             includeString('command', args.command, 2000);
             break;
+        case 'request_takeover':
+            includeString('reason', args.reason, 500);
+            break;
+        case 'computer_act':
+            if (Array.isArray(args.actions)) {
+                metadata.actionCount = args.actions.length;
+                includeString('actions', args.actions
+                    .map(action => action && typeof action === 'object'
+                    ? String(action.type ?? '?')
+                    : '?')
+                    .join(','), 500);
+            }
+            break;
         default:
             break;
     }
     return metadata;
 }
+/**
+ * Screenshots the live model context keeps (rakazo's trick): the newest
+ * screenshot-bearing tool results retain their images, everything older
+ * drops back to its text observation. Bounds tokens and sidesteps most of
+ * the cross-run screenshot persistence problem — a resumed run re-observes.
+ */
+const WORK_LIVE_SCREENSHOT_MESSAGE_LIMIT = 2;
+function retainRecentWorkImages(messages, limit = WORK_LIVE_SCREENSHOT_MESSAGE_LIMIT) {
+    let retained = 0;
+    for (let index = messages.length - 1; index >= 0; index--) {
+        const message = messages[index];
+        if (!message.images?.length)
+            continue;
+        retained += 1;
+        if (retained > limit)
+            delete message.images;
+    }
+}
+function computerObservationDiff(observation, computerContext) {
+    const previous = computerContext?.previous;
+    if (computerContext) {
+        computerContext.previous = {
+            ...(observation.screenshotSha256
+                ? { screenshotSha256: observation.screenshotSha256 }
+                : {}),
+            window: observation.window,
+            ...(observation.url ? { url: observation.url } : {}),
+            ...(observation.pageFocus !== undefined
+                ? { pageFocus: observation.pageFocus }
+                : {}),
+        };
+    }
+    if (!previous?.screenshotSha256 || !observation.screenshotSha256)
+        return [];
+    if (previous.screenshotSha256 === observation.screenshotSha256) {
+        return [
+            'Since the previous observation: the screen is IDENTICAL — nothing changed visually.',
+        ];
+    }
+    const transitions = [
+        ...(previous.window !== observation.window
+            ? [`window "${previous.window}" → "${observation.window}"`]
+            : []),
+        ...(previous.url !== observation.url
+            ? [`URL ${previous.url || '(none)'} → ${observation.url || '(none)'}`]
+            : []),
+        ...(previous.pageFocus !== observation.pageFocus
+            ? [`page focus ${previous.pageFocus} → ${observation.pageFocus}`]
+            : []),
+    ];
+    return [
+        `Since the previous observation: the screen changed${transitions.length ? ` (${transitions.join('; ')})` : ' (same window, title, and URL)'}.`,
+    ];
+}
+function computerToolResult(observation, actionCount, computerContext) {
+    const ranCount = observation.fence
+        ? observation.fence.afterAction
+        : actionCount;
+    const browserChrome = observation.pageFocus === false && /chromium/i.test(observation.window);
+    const diffLines = computerObservationDiff(observation, computerContext);
+    const receiptLines = [
+        ...(observation.clickReceipts?.length
+            ? [
+                `Click receipts: ${observation.clickReceipts
+                    .map(receipt => `#${receipt.action} (${receipt.x},${receipt.y}) ${receipt.changed ? 'changed nearby pixels' : 'NO visible change nearby'}`)
+                    .join('; ')}.`,
+            ]
+            : []),
+        ...(observation.scrollReceipts?.map(receipt => receipt.found === null
+            ? `Scroll receipt: action #${receipt.action} scrolled ${receipt.scrolledUnits} unit(s) — no semantic signal, target visibility unknown.`
+            : receipt.found && receipt.visible
+                ? `Scroll receipt: action #${receipt.action} scrolled ${receipt.scrolledUnits} unit(s) — target is now visible.`
+                : receipt.found
+                    ? `Scroll receipt: action #${receipt.action} scrolled ${receipt.scrolledUnits} unit(s) — target found but still not fully visible.`
+                    : `Scroll receipt: action #${receipt.action} scrolled ${receipt.scrolledUnits} unit(s) — target NOT found in the scrolled range.`) ?? []),
+    ];
+    const summary = [
+        ...(observation.fence
+            ? [`BATCH STOPPED EARLY: ${observation.fence.detail}`]
+            : []),
+        ...(actionCount !== undefined
+            ? [
+                `Applied ${ranCount} of ${actionCount} action${actionCount === 1 ? '' : 's'} and captured the settled screen.`,
+            ]
+            : []),
+        `Screen ${observation.width}x${observation.height}, cursor at ${observation.cursorX},${observation.cursorY}.`,
+        observation.window
+            ? `Active window: ${observation.window}` +
+                (observation.windowId !== undefined
+                    ? ` (#${observation.windowId}${observation.windowCount !== undefined ? `, ${observation.windowCount} visible` : ''})`
+                    : '')
+            : 'No active window.',
+        ...(observation.url ? [`Page URL: ${observation.url}`] : []),
+        ...(browserChrome
+            ? [
+                'Keyboard focus: the browser UI, NOT the page — typed text would go to the omnibox or a browser dialog.',
+            ]
+            : observation.focusedElement
+                ? [`Focused element: ${observation.focusedElement}`]
+                : []),
+        ...(observation.expect
+            ? [
+                observation.expect.outcome === 'passed'
+                    ? 'Declared expectation: PASSED.'
+                    : `Declared expectation: NOT yet observed (pending${observation.expect.unmet?.length ? `: ${observation.expect.unmet.join(', ')}` : ''}) — re-observe before assuming failure.`,
+            ]
+            : []),
+        ...receiptLines,
+        ...diffLines,
+        ...(observation.screenshotSha256
+            ? [`Screenshot sha256: ${observation.screenshotSha256.slice(0, 16)}`]
+            : []),
+        'The screenshot accompanies this result.',
+    ].join('\n');
+    return {
+        content: summary,
+        metadata: {
+            screenshot: true,
+            width: observation.width,
+            height: observation.height,
+            cursorX: observation.cursorX,
+            cursorY: observation.cursorY,
+            window: observation.window.slice(0, 300),
+            ...(observation.windowId !== undefined
+                ? { windowId: observation.windowId }
+                : {}),
+            ...(observation.windowCount !== undefined
+                ? { windowCount: observation.windowCount }
+                : {}),
+            ...(observation.url ? { url: observation.url } : {}),
+            ...(observation.pageFocus !== undefined
+                ? { pageFocus: observation.pageFocus }
+                : {}),
+            ...(observation.focusedElement
+                ? { focusedElement: observation.focusedElement }
+                : {}),
+            ...(observation.screenshotSha256
+                ? { screenshotSha256: observation.screenshotSha256 }
+                : {}),
+            ...(observation.fence ? { fence: observation.fence } : {}),
+            ...(observation.expect ? { expect: observation.expect } : {}),
+            ...(observation.clickReceipts?.length
+                ? { clickReceipts: observation.clickReceipts }
+                : {}),
+            ...(observation.scrollReceipts?.length
+                ? { scrollReceipts: observation.scrollReceipts }
+                : {}),
+            ...(actionCount !== undefined ? { actionCount } : {}),
+            ...(ranCount !== undefined && ranCount !== actionCount
+                ? { ranCount }
+                : {}),
+        },
+        images: [observation.screenshotBase64],
+    };
+}
 function boundPersistedToolOutput(value) {
     const maxLength = 20000;
     if (value.length <= maxLength)