@namzu/sdk 45.1.0 → 48.0.0

package/dist/tools/builtins/computer-use.js

@@ -1,95 +1,217 @@
 import { z } from 'zod';
+import { resolveProviderCapabilities } from '../../provider/capabilities.js';
+import { sleep } from '../../utils/backoff.js';
 import { defineTool } from '../defineTool.js';
+import { neutralizeEnvelopeDelimiter, wrapUntrusted } from '../untrusted-envelope.js';
+import { ScreenshotFrames, assumedDisplay, desktopRectOnImage, pointOnImage, toDisplayPoint, toDisplayRect, toImagePoint, } from './computer-use-coordinates.js';
+import { STANDARD_SCREENSHOT_LIMITS, cropAndFitPng, fitPng, pngSize, } from './computer-use-image.js';
+export { HIGH_RES_SCREENSHOT_LIMITS, STANDARD_SCREENSHOT_LIMITS, screenshotTargetSize, } from './computer-use-image.js';
 export const COMPUTER_USE_TOOL_NAME = 'computer_use';
+const DEFAULT_SETTLE_MS = 500;
+const DEFAULT_MAX_BATCH_ACTIONS = 20;
+const DEFAULT_MAX_WAIT_MS = 10_000;
+const MAX_LISTED_WINDOWS = 50;
+/** The most of a ui_snapshot the model is shown, in characters (about 4 000 tokens). */
+const MAX_UI_SNAPSHOT_CHARS = 14_000;
+/**
+ * Why `provider` cannot drive `computer_use`, or undefined when it can.
+ *
+ * The tool's only way to show the model the screen is an image in a tool
+ * result. A driver that declares `supportsToolResultImages: false` replaces
+ * that image with a line of text, so the model acts on a screen it has never
+ * seen while every call reports success. Pass the answer to
+ * {@link ComputerUseToolOptions.unavailableReason}.
+ */
+export function computerUseUnavailableReason(provider) {
+    if (resolveProviderCapabilities(provider).supportsToolResultImages)
+        return undefined;
+    return `The ${provider.id} provider cannot return images in tool results, so the model would never see a screenshot. Use a provider that can (for example Anthropic, Codex or Google) for computer use.`;
+}
 // ---------------------------------------------------------------------------
-// Input schema — discriminated union matching ComputerUseAction
+// Input schema
 // ---------------------------------------------------------------------------
 const pointSchema = z.object({
     x: z.number().int(),
     y: z.number().int(),
 });
-const mouseButtonSchema = z.enum(['left', 'right', 'middle']);
+const regionSchema = z.object({
+    x: z.number().int(),
+    y: z.number().int(),
+    width: z.number().int().positive(),
+    height: z.number().int().positive(),
+});
+// Left when omitted: a model asked to "click the Start button" often leaves
+// the button out, and refusing that cost a whole round trip for nothing.
+const mouseButtonSchema = z.enum(['left', 'right', 'middle']).default('left');
+const screenshotIdSchema = z.string().min(1).optional();
+const cursorPositionSchema = z.object({ type: z.literal('cursor_position') });
+const mouseMoveSchema = z.object({ type: z.literal('mouse_move'), to: pointSchema });
+const mouseClickSchema = z.object({
+    type: z.literal('mouse_click'),
+    at: pointSchema,
+    button: mouseButtonSchema,
+});
+const mouseDragSchema = z.object({
+    type: z.literal('mouse_drag'),
+    from: pointSchema,
+    to: pointSchema,
+    button: mouseButtonSchema,
+});
+const scrollSchema = z.object({
+    type: z.literal('scroll'),
+    at: pointSchema,
+    direction: z.enum(['up', 'down', 'left', 'right']),
+    amount: z.number().int().positive(),
+});
+const typeTextSchema = z.object({ type: z.literal('type_text'), text: z.string() });
+const keySchema = z.object({ type: z.literal('key'), keys: z.string() });
+const waitSchema = z.object({ type: z.literal('wait'), ms: z.number().int().nonnegative() });
+const listWindowsSchema = z.object({ type: z.literal('list_windows') });
+const focusWindowSchema = z.object({
+    type: z.literal('focus_window'),
+    window_id: z.string().min(1),
+});
+const UI_ACTIONS = [
+    'invoke',
+    'set_value',
+    'toggle',
+    'select',
+    'expand',
+    'collapse',
+    'focus',
+    'scroll_into_view',
+];
+const uiSnapshotSchema = z.object({
+    type: z.literal('ui_snapshot'),
+    window_id: z.string().min(1).optional(),
+});
+const uiActSchema = z.object({
+    type: z.literal('ui_act'),
+    ref: z.string().min(1),
+    action: z.enum(UI_ACTIONS),
+    value: z.string().optional(),
+});
+/** What a batch may carry: everything but the image-returning actions and another batch. */
+const batchItemSchema = z.discriminatedUnion('type', [
+    cursorPositionSchema,
+    mouseMoveSchema,
+    mouseClickSchema,
+    mouseDragSchema,
+    scrollSchema,
+    typeTextSchema,
+    keySchema,
+    waitSchema,
+    listWindowsSchema,
+    focusWindowSchema,
+    uiActSchema,
+]);
+const withFrame = { screenshot_id: screenshotIdSchema };
 const actionSchema = z.discriminatedUnion('type', [
     z.object({ type: z.literal('screenshot') }),
-    z.object({ type: z.literal('cursor_position') }),
-    z.object({ type: z.literal('mouse_move'), to: pointSchema }),
-    z.object({ type: z.literal('mouse_click'), at: pointSchema, button: mouseButtonSchema }),
-    z.object({
-        type: z.literal('mouse_drag'),
-        from: pointSchema,
-        to: pointSchema,
-        button: mouseButtonSchema,
-    }),
+    z.object({ type: z.literal('zoom'), region: regionSchema, ...withFrame }),
+    cursorPositionSchema.extend(withFrame),
+    mouseMoveSchema.extend(withFrame),
+    mouseClickSchema.extend(withFrame),
+    mouseDragSchema.extend(withFrame),
+    scrollSchema.extend(withFrame),
+    typeTextSchema.extend(withFrame),
+    keySchema.extend(withFrame),
+    waitSchema.extend(withFrame),
+    listWindowsSchema.extend(withFrame),
+    focusWindowSchema.extend(withFrame),
+    uiSnapshotSchema,
+    uiActSchema.extend(withFrame),
     z.object({
-        type: z.literal('scroll'),
-        at: pointSchema,
-        direction: z.enum(['up', 'down', 'left', 'right']),
-        amount: z.number().int().positive(),
+        type: z.literal('batch'),
+        actions: z.array(batchItemSchema).min(1),
+        ...withFrame,
     }),
-    z.object({ type: z.literal('type_text'), text: z.string() }),
-    z.object({ type: z.literal('key'), keys: z.string() }),
 ]);
-/**
- * The provider-facing shape is deliberately flat.
- *
- * The runtime schema above is the authoritative contract: it knows which
- * fields each action requires. Rendering that discriminated union produces a
- * root `anyOf`, however, and some custom-tool wires reject root combinators
- * even when every branch is an object. A model can still see every field and
- * every action here; incomplete combinations are rejected by `actionSchema`
- * before the host is called, with the recovery hint below.
- */
-const pointModelInputSchema = {
-    type: 'object',
-    properties: {
-        x: { type: 'integer' },
-        y: { type: 'integer' },
-    },
-    required: ['x', 'y'],
-    additionalProperties: false,
-};
-const modelInputSchema = {
-    type: 'object',
-    properties: {
-        type: {
-            type: 'string',
-            enum: [
-                'screenshot',
-                'cursor_position',
-                'mouse_move',
-                'mouse_click',
-                'mouse_drag',
-                'scroll',
-                'type_text',
-                'key',
-            ],
-            description: 'Desktop action. screenshot and cursor_position need no other fields; mouse_move needs to; mouse_click needs at and button; mouse_drag needs from, to, and button; scroll needs at, direction, and amount; type_text needs text; key needs keys.',
-        },
-        to: pointModelInputSchema,
-        at: pointModelInputSchema,
-        from: pointModelInputSchema,
-        button: { type: 'string', enum: ['left', 'right', 'middle'] },
-        direction: { type: 'string', enum: ['up', 'down', 'left', 'right'] },
-        amount: { type: 'integer', description: 'Positive integer scroll distance.' },
-        text: { type: 'string', description: 'Literal text to type.' },
-        keys: {
-            type: 'string',
-            description: 'Key or key chord to press, for example ENTER or CTRL+R.',
-        },
-    },
-    required: ['type'],
-    additionalProperties: false,
-};
-const DESTRUCTIVE_ACTION_TYPES = new Set([
+const HOST_ACTIONS = [
+    'screenshot',
+    'cursor_position',
+    'mouse_move',
     'mouse_click',
     'mouse_drag',
+    'scroll',
     'type_text',
     'key',
+];
+const ALL_ACTIONS = [
+    'screenshot',
+    'zoom',
+    'cursor_position',
+    'mouse_move',
+    'mouse_click',
+    'mouse_drag',
     'scroll',
+    'type_text',
+    'key',
+    'wait',
+    'list_windows',
+    'focus_window',
+    'ui_snapshot',
+    'ui_act',
+    'batch',
+];
+/** Observations: they change nothing on the desktop. */
+const READ_ONLY_ACTIONS = new Set([
+    'screenshot',
+    'zoom',
+    'cursor_position',
+    'wait',
+    'list_windows',
+    'ui_snapshot',
 ]);
+const DESTRUCTIVE_ACTIONS = new Set([
+    'mouse_click',
+    'mouse_drag',
+    'type_text',
+    'key',
+    'scroll',
+    'ui_act',
+]);
+/** Observations that show the model what is on the screen, on their own. */
+const SCREEN_OBSERVATIONS = new Set(['screenshot', 'zoom', 'list_windows', 'ui_snapshot']);
+/**
+ * Terminal applications, by the process name a host reports in
+ * `WindowInfo.app` (on Windows without `.exe`). Keys are not typed into these.
+ */
+const TERMINAL_APPS = /^(WindowsTerminal|OpenConsole|conhost|cmd|powershell|pwsh|wsl|mintty|wezterm(-gui)?|alacritty|Hyper|Tabby|kitty|ConEmu(64)?|putty|Terminal|iTerm2?|gnome-terminal(-server)?|konsole|xterm|xfce4-terminal|tilix|terminator|foot|ghostty|Warp)$/i;
+/** Actions a batch cannot carry: the ones that return an image or a tree, and batch itself. */
+const OUTSIDE_BATCH = new Set(['screenshot', 'zoom', 'ui_snapshot', 'batch']);
+/** A single action's whole result when it only acted: `<label>: done`, then its screenshot. */
+const ACKNOWLEDGEMENT = /^[^\n]*: done(\nScreenshot s\d+ \(\d+x\d+\)\.)?$/;
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function batchActions(input) {
+    if (!isRecord(input) || input.type !== 'batch')
+        return null;
+    return Array.isArray(input.actions) ? input.actions : [];
+}
+/** Read-only when every action is: a batch with one click in it is not. Unknown shapes are not. */
+function isReadOnlyInput(input) {
+    const actions = batchActions(input);
+    if (actions)
+        return (actions.length > 0 &&
+            actions.every((item) => isRecord(item) && READ_ONLY_ACTIONS.has(String(item.type))));
+    return isRecord(input) && READ_ONLY_ACTIONS.has(String(input.type));
+}
+function isDestructiveInput(input) {
+    const actions = batchActions(input);
+    if (actions)
+        return actions.some((item) => !isRecord(item) || DESTRUCTIVE_ACTIONS.has(String(item.type)));
+    return isRecord(input) && DESTRUCTIVE_ACTIONS.has(String(input.type));
+}
+// ---------------------------------------------------------------------------
+// Capabilities
+// ---------------------------------------------------------------------------
 function requiredCapability(type) {
     switch (type) {
         case 'screenshot':
+        case 'zoom':
+        case 'wait':
             return 'screenshot';
         case 'cursor_position':
             return 'cursorPosition';
@@ -101,13 +223,53 @@ function requiredCapability(type) {
         case 'type_text':
         case 'key':
             return 'keyboard';
+        case 'list_windows':
+        case 'focus_window':
+            return 'windows';
+        case 'ui_snapshot':
+        case 'ui_act':
+            return 'uiTree';
         default:
             return null;
     }
 }
-function buildDescription(host) {
-    const caps = host.capabilities;
-    const available = availableActions(caps);
+function hostActionAvailable(caps, action) {
+    const required = requiredCapability(action);
+    return ((required === null || caps[required] === true) &&
+        (caps.supportedActions === undefined || caps.supportedActions.includes(action)) &&
+        (action !== 'mouse_click' || caps.mouseClickButtons?.length !== 0) &&
+        (action !== 'mouse_drag' || caps.mouseDragButtons?.length !== 0));
+}
+function availableActions(host, caps) {
+    const screenshot = hostActionAvailable(caps, 'screenshot');
+    const windows = caps.windows === true &&
+        typeof host.listWindows === 'function' &&
+        typeof host.focusWindow === 'function';
+    const uiTree = caps.uiTree === true &&
+        typeof host.uiSnapshot === 'function' &&
+        typeof host.uiAct === 'function';
+    const available = ALL_ACTIONS.filter((action) => {
+        switch (action) {
+            case 'zoom':
+            case 'wait':
+                return screenshot;
+            case 'list_windows':
+            case 'focus_window':
+                return windows;
+            case 'ui_snapshot':
+            case 'ui_act':
+                return uiTree;
+            case 'batch':
+                return false;
+            default:
+                return hostActionAvailable(caps, action);
+        }
+    });
+    if (available.some((action) => !OUTSIDE_BATCH.has(action)))
+        available.push('batch');
+    return available;
+}
+function unavailableHostActions(caps) {
     const unavailable = [];
     if (!caps.screenshot)
         unavailable.push('screenshot');
@@ -118,13 +280,21 @@ function buildDescription(host) {
     if (!caps.keyboard)
         unavailable.push('keyboard');
     if (caps.supportedActions) {
-        for (const action of actionSchema.options.map((option) => option.shape.type.value)) {
-            if (!available.includes(action))
+        for (const action of HOST_ACTIONS) {
+            if (!hostActionAvailable(caps, action) && !unavailable.includes(action))
                 unavailable.push(action);
         }
     }
+    return unavailable;
+}
+// ---------------------------------------------------------------------------
+// Model-facing description and schema
+// ---------------------------------------------------------------------------
+function buildDescription(host, caps, settings) {
+    const available = availableActions(host, caps);
+    const unavailable = unavailableHostActions(caps);
     const lines = [
-        `Controls the user's desktop on a ${caps.displayServer} host. Use to take screenshots and drive mouse/keyboard input for GUI tasks.`,
+        `Controls the user's desktop on a ${caps.displayServer} host: screenshots, mouse and keyboard, for GUI tasks.`,
         `Available actions: ${available.join('; ') || 'none'}.`,
     ];
     if (unavailable.length > 0) {
@@ -132,31 +302,79 @@ function buildDescription(host) {
             ? `Unavailable on this host: ${unavailable.join(', ')} — ${caps.unavailableReason} Do not retry; tell the user.`
             : `Unavailable on this host: ${unavailable.join(', ')}.`);
     }
-    lines.push('Coordinates are in logical pixels from the top-left of the primary display. Call getDisplayGeometry through screenshot output before clicking to confirm bounds.');
+    if (!available.includes('screenshot'))
+        return finish(lines, caps, available);
+    lines.push('Coordinates: every x/y you send is a pixel of the most recent screenshot this tool returned — origin at its top-left, x to the right, y down — never a screen pixel. Each screenshot states its id and size (for example "s3: 1456x819"); stay inside it. The tool maps your coordinates onto the display. To aim at an earlier screenshot, pass its id as screenshot_id.', 'Take a screenshot before your first click.');
+    if (settings.screenshotAfterActions)
+        lines.push(`After any action that changes something, the tool waits ${settings.settleMs} ms and returns a new screenshot — do not call screenshot again after acting.`);
+    lines.push('zoom returns a closer, sharper view of region {x, y, width, height} of the screenshot; your coordinates still refer to the screenshot, not to the zoomed image.', `wait pauses for ms milliseconds (at most ${settings.maxWaitMs} per call, a batch's waits together) and then shows the screen.`);
+    if (available.includes('batch'))
+        lines.push(`batch: {"type":"batch","actions":[...]} runs up to ${settings.maxBatchActions} actions in order, stops at the first one that fails, and returns one screenshot at the end. Every call costs a model round trip of several seconds, so put the steps you can already predict into one batch (click a field, type, press ENTER) rather than one call each — but end the batch at any step that should bring up a new window, and look before typing into it. A batch cannot contain screenshot${available.includes('ui_snapshot') ? ', zoom or ui_snapshot' : ' or zoom'}.`);
+    if (caps.displayServer === 'win32' &&
+        available.includes('key') &&
+        available.includes('type_text'))
+        lines.push('To start a Windows program, a shell command (Start-Process notepad, or cmd.exe /c start calc) is surest when you have a shell tool. The Start menu searches by display names in the system language, and ENTER on a name it does not find opens a web search in the browser.');
+    if (available.includes('list_windows'))
+        lines.push('list_windows names the open windows with their ids; focus_window brings one to the front.');
+    if (available.includes('ui_snapshot'))
+        lines.push(`ui_snapshot {window_id} reads a window's controls (its accessibility tree) as text, each control you can act on with a ref such as e12; without window_id it reads the window in front. ui_act {ref, action, value} acts on one control: invoke (press a button, open a menu item), set_value (replace a field's text with value), toggle, select, expand, collapse. Prefer ui_act to clicking pixels when the control is in the tree: it does not depend on coordinates or on which window is in front, and a batch of ui_act steps (for example pressing several buttons) runs in one call. Refs are valid only until the next ui_snapshot; take a new one after the window changes.`);
+    lines.push('Typing and keys go to whichever window has focus: confirm on a screenshot that the right window is in front before type_text or key.');
+    if (available.includes('list_windows'))
+        lines.push('type_text and key are refused while a terminal window is in front — usually the one running this agent, or the user’s own; bring the window you mean to the front first (focus_window, or click it).');
+    return finish(lines, caps, available);
+}
+function finish(lines, caps, available) {
     if (caps.mouseClickButtons && available.includes('mouse_click'))
         lines.push(`Click buttons: ${caps.mouseClickButtons.join(', ') || 'none'}.`);
     if (caps.mouseDragButtons && available.includes('mouse_drag'))
         lines.push(`Drag buttons: ${caps.mouseDragButtons.join(', ') || 'none'}.`);
     return lines.join(' ');
 }
-function availableActions(caps) {
-    return actionSchema.options
-        .map((option) => option.shape.type.value)
-        .filter((action) => {
-        const required = requiredCapability(action);
-        return ((required === null || caps[required] === true) &&
-            (caps.supportedActions === undefined || caps.supportedActions.includes(action)) &&
-            (action !== 'mouse_click' || caps.mouseClickButtons?.length !== 0) &&
-            (action !== 'mouse_drag' || caps.mouseDragButtons?.length !== 0));
-    });
+/**
+ * The provider-facing shape is deliberately flat.
+ *
+ * The runtime schema above is the authoritative contract: it knows which
+ * fields each action requires. Rendering that discriminated union produces a
+ * root `anyOf`, however, and some custom-tool wires reject root combinators
+ * even when every branch is an object. A model can still see every field and
+ * every action here; incomplete combinations are rejected by `actionSchema`
+ * before the host is called, with the recovery hint below.
+ */
+function pointModelSchema(description) {
+    return {
+        type: 'object',
+        ...(description ? { description } : {}),
+        properties: {
+            x: { type: 'integer', description: 'Horizontal pixel coordinate, from the left edge.' },
+            y: { type: 'integer', description: 'Vertical pixel coordinate, from the top edge.' },
+        },
+        required: ['x', 'y'],
+        additionalProperties: false,
+    };
 }
-function hostModelSchema(caps) {
-    const schema = structuredClone(modelInputSchema);
-    const actions = availableActions(caps);
+const ACTION_REQUIREMENTS = {
+    screenshot: 'screenshot needs no other fields',
+    zoom: 'zoom needs region',
+    cursor_position: 'cursor_position needs no other fields',
+    mouse_move: 'mouse_move needs to',
+    mouse_click: 'mouse_click needs at (button defaults to left)',
+    mouse_drag: 'mouse_drag needs from and to (button defaults to left)',
+    scroll: 'scroll needs at, direction, and amount',
+    type_text: 'type_text needs text',
+    key: 'key needs keys',
+    wait: 'wait needs ms',
+    list_windows: 'list_windows needs no other fields',
+    focus_window: 'focus_window needs window_id',
+    ui_snapshot: 'ui_snapshot takes an optional window_id',
+    ui_act: 'ui_act needs ref and action, and value for set_value',
+    batch: 'batch needs actions',
+};
+function hostModelSchema(host, caps, settings) {
     // An unavailable host remains a diagnostic tool. Avoid invalid empty enums
     // on provider wires; every execution is still refused before host access.
-    if (actions.length > 0)
-        schema.properties.type.enum = actions;
+    const offered = availableActions(host, caps);
+    const actions = offered.length > 0 ? offered : ALL_ACTIONS.filter((a) => a !== 'batch');
+    const items = actions.filter((action) => !OUTSIDE_BATCH.has(action));
     const buttons = new Set();
     if (actions.includes('mouse_click'))
         for (const button of caps.mouseClickButtons ?? ['left', 'right', 'middle'])
@@ -164,10 +382,101 @@ function hostModelSchema(caps) {
     if (actions.includes('mouse_drag'))
         for (const button of caps.mouseDragButtons ?? ['left', 'right', 'middle'])
             buttons.add(button);
-    if (buttons.size > 0)
-        schema.properties.button.enum = [...buttons];
-    return schema;
+    const buttonEnum = buttons.size > 0 ? [...buttons] : ['left', 'right', 'middle'];
+    const fieldSchemas = (forItems) => {
+        const present = forItems ? items : actions;
+        const fields = {
+            to: pointModelSchema('Destination point: where mouse_move moves the cursor to, or where mouse_drag releases.'),
+            at: pointModelSchema('Point the action happens at: where to click, or where to scroll.'),
+            from: pointModelSchema('Drag start point, for mouse_drag.'),
+            button: { type: 'string', enum: buttonEnum, description: 'Mouse button; left when omitted.' },
+            direction: {
+                type: 'string',
+                enum: ['up', 'down', 'left', 'right'],
+                description: 'Which way to scroll.',
+            },
+            amount: { type: 'integer', description: 'Positive integer scroll distance.' },
+            text: { type: 'string', description: 'Literal text to type.' },
+            keys: {
+                type: 'string',
+                description: 'Key or key chord to press, for example ENTER or CTRL+R.',
+            },
+            ms: {
+                type: 'integer',
+                description: `Milliseconds to wait, 0 to ${settings.maxWaitMs}.`,
+            },
+        };
+        if (present.includes('focus_window') || (!forItems && present.includes('ui_snapshot')))
+            fields.window_id = { type: 'string', description: 'A window id from list_windows.' };
+        if (present.includes('ui_act')) {
+            fields.ref = {
+                type: 'string',
+                description: 'A ref from the latest ui_snapshot, such as e12.',
+            };
+            fields.action = {
+                type: 'string',
+                enum: [...UI_ACTIONS],
+                description: 'What ui_act does to the control.',
+            };
+            fields.value = { type: 'string', description: 'The text set_value puts in the control.' };
+        }
+        if (!forItems && present.includes('zoom'))
+            fields.region = {
+                type: 'object',
+                description: 'Area of the screenshot to zoom into, in its pixels.',
+                properties: {
+                    x: { type: 'integer', description: 'Left edge of the region, in screenshot pixels.' },
+                    y: { type: 'integer', description: 'Top edge of the region, in screenshot pixels.' },
+                    width: { type: 'integer', description: 'Width of the region, in screenshot pixels.' },
+                    height: { type: 'integer', description: 'Height of the region, in screenshot pixels.' },
+                },
+                required: ['x', 'y', 'width', 'height'],
+                additionalProperties: false,
+            };
+        return fields;
+    };
+    const properties = {
+        type: {
+            type: 'string',
+            enum: actions,
+            description: `Desktop action. ${actions.map((action) => ACTION_REQUIREMENTS[action]).join('; ')}.`,
+        },
+        ...fieldSchemas(false),
+    };
+    if (actions.includes('batch'))
+        properties.actions = {
+            type: 'array',
+            minItems: 1,
+            maxItems: settings.maxBatchActions,
+            description: `For type batch: up to ${settings.maxBatchActions} actions run in order (no screenshot, zoom, ui_snapshot or batch inside).`,
+            items: {
+                type: 'object',
+                properties: {
+                    type: {
+                        type: 'string',
+                        enum: items,
+                        description: 'This batched action.',
+                    },
+                    ...fieldSchemas(true),
+                },
+                required: ['type'],
+                additionalProperties: false,
+            },
+        };
+    properties.screenshot_id = {
+        type: 'string',
+        description: 'Optional id of the screenshot your coordinates were read from, such as s3. Defaults to the latest.',
+    };
+    return {
+        type: 'object',
+        properties,
+        required: ['type'],
+        additionalProperties: false,
+    };
 }
+// ---------------------------------------------------------------------------
+// Labels
+// ---------------------------------------------------------------------------
 function pointLabel(point) {
     return `(${point.x}, ${point.y})`;
 }
@@ -176,11 +485,41 @@ function quotedText(value) {
     const visible = oneLine.length > 64 ? `${oneLine.slice(0, 63)}…` : oneLine;
     return JSON.stringify(visible);
 }
-/** Human activity text; the raw action union remains the model-facing input. */
-function actionLabel(input) {
+function waitLabel(ms) {
+    return ms >= 1000 ? `Wait ${Number((ms / 1000).toFixed(1))} s` : `Wait ${ms} ms`;
+}
+/** What a ui_act does, as a verb phrase over the control's description. */
+function uiActLabel(input, describe) {
+    const target = describe?.(input.ref) ?? input.ref;
+    switch (input.action) {
+        case 'invoke':
+            return `Press ${target}`;
+        case 'set_value':
+            return `Set ${target} to ${quotedText(input.value ?? '')}`;
+        case 'toggle':
+            return `Toggle ${target}`;
+        case 'select':
+            return `Select ${target}`;
+        case 'expand':
+            return `Expand ${target}`;
+        case 'collapse':
+            return `Collapse ${target}`;
+        case 'focus':
+            return `Focus ${target}`;
+        case 'scroll_into_view':
+            return `Scroll to ${target}`;
+    }
+}
+/**
+ * Human activity text for one action; the raw input remains the model-facing
+ * record. `describe` names a ui_act ref's control when the tool knows it.
+ */
+function itemLabel(input, describe) {
     switch (input.type) {
         case 'screenshot':
             return 'Capture screenshot';
+        case 'zoom':
+            return `Zoom into ${input.region.width}x${input.region.height} at ${pointLabel(input.region)}`;
         case 'cursor_position':
             return 'Read cursor position';
         case 'mouse_move':
@@ -195,33 +534,49 @@ function actionLabel(input) {
             return `Type ${quotedText(input.text)}`;
         case 'key':
             return `Press ${input.keys}`;
+        case 'wait':
+            return waitLabel(input.ms);
+        case 'list_windows':
+            return 'List windows';
+        case 'focus_window':
+            return `Focus window ${input.window_id}`;
+        case 'ui_snapshot':
+            return input.window_id
+                ? `Read the controls of window ${input.window_id}`
+                : 'Read the controls of the front window';
+        case 'ui_act':
+            return uiActLabel(input, describe);
+        case 'batch':
+            return batchLabel(input.actions, describe);
     }
 }
-function resultToToolResult(result) {
-    switch (result.type) {
-        case 'screenshot': {
-            const { data, mimeType, width, height } = result.result;
-            // `output` used to BE the base64 payload, which meant the model
-            // received 400 KB–2.7 MB of undecodable characters as text —
-            // roughly 100k–670k tokens — and could not see the screen at
-            // all. The image now travels as a content block; `output` keeps
-            // the short human/transcript-facing description.
-            return {
-                success: true,
-                output: `Screenshot captured (${width}x${height}, ${mimeType}).`,
-                content: [{ type: 'image', data: data.toString('base64'), mediaType: mimeType }],
-                data: { mimeType, width, height, encoding: 'base64' },
-            };
-        }
-        case 'cursor_position':
-            return {
-                success: true,
-                output: JSON.stringify(result.point),
-                data: result.point,
-            };
-        case 'ok':
-            return { success: true, output: 'ok' };
-    }
+function batchLabel(actions, describe) {
+    const parts = actions.map((item) => itemLabel(item, describe));
+    const head = `${actions.length} desktop action${actions.length === 1 ? '' : 's'}`;
+    const joined = parts.join(' · ');
+    return joined.length > 240 ? `${head}: ${joined.slice(0, 239)}…` : `${head}: ${joined}`;
+}
+function resolveSettings(options) {
+    const count = (value, fallback, min, name) => {
+        if (value === undefined)
+            return fallback;
+        if (!Number.isSafeInteger(value) || value < min)
+            throw new RangeError(`createComputerUseTool: ${name} must be an integer ≥ ${min}`);
+        return value;
+    };
+    const limits = options.screenshotLimits ?? STANDARD_SCREENSHOT_LIMITS;
+    if (!Number.isSafeInteger(limits.maxLongEdge) ||
+        !Number.isSafeInteger(limits.maxTiles) ||
+        limits.maxLongEdge < 28 ||
+        limits.maxTiles < 1)
+        throw new RangeError('createComputerUseTool: screenshotLimits needs an integer maxLongEdge ≥ 28 and maxTiles ≥ 1');
+    return {
+        limits,
+        settleMs: count(options.settleMs, DEFAULT_SETTLE_MS, 0, 'settleMs'),
+        screenshotAfterActions: options.screenshotAfterActions ?? true,
+        maxBatchActions: count(options.maxBatchActions, DEFAULT_MAX_BATCH_ACTIONS, 1, 'maxBatchActions'),
+        maxWaitMs: count(options.maxWaitMs, DEFAULT_MAX_WAIT_MS, 0, 'maxWaitMs'),
+    };
 }
 function isOutcomeUnknown(value, action) {
     if (typeof value !== 'object' || value === null)
@@ -237,30 +592,81 @@ function isOutcomeUnknown(value, action) {
         typeof candidate.message === 'string' &&
         candidate.message.length > 0);
 }
-function unknownOutcomeToToolResult(error) {
-    return {
-        success: false,
-        output: '',
-        error: error.message,
-        data: {
-            code: error.code,
-            action: error.action,
-            outcome: error.outcome,
-            retrySafety: error.retrySafety,
-            timedOut: error.timedOut,
-            exitCode: error.exitCode,
-        },
-    };
+function errorText(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+class StepFailure extends Error {
+    unknown;
+    constructor(message, unknown) {
+        super(message);
+        this.unknown = unknown;
+    }
+}
+/** Whether a call sends the screen to the provider (see `ToolDefinition.capturesScreen`). */
+function capturesScreenInput(input, screenshotAfterActions) {
+    const items = batchActions(input) ?? [input];
+    return items.some((item) => {
+        if (!isRecord(item))
+            return true;
+        const type = String(item.type);
+        if (SCREEN_OBSERVATIONS.has(type))
+            return true;
+        if (type === 'cursor_position')
+            return false;
+        // Everything else is followed by a screenshot unless that is switched off.
+        return screenshotAfterActions;
+    });
+}
+function safeRole(role) {
+    return /^[A-Za-z][\w-]{0,39}$/.test(role) ? role : 'Element';
+}
+/** `Button "Beş"`: the control's role and name, one line, cut to fit. */
+function uiElementText(element) {
+    const name = element.name.trim();
+    return name.length > 0 ? `${safeRole(element.role)} ${quotedText(name)}` : safeRole(element.role);
+}
+/**
+ * One line of a ui_snapshot: `[e12] Button "Beş" (disabled) [invoke] @(212, 488)`.
+ * The ref and position are the tool's; the rest is the application's, and
+ * sits inside the untrusted frame.
+ */
+function uiElementLine(element, ref, frame) {
+    const parts = [`${ref ? `[${ref}] ` : ''}${uiElementText(element)}`];
+    if (element.value !== undefined && element.value !== element.name)
+        parts.push(`value=${quotedText(element.value)}`);
+    const states = (element.states ?? []).filter((state) => /^[a-z_]{1,24}$/.test(state));
+    if (states.length > 0)
+        parts.push(`(${states.join(', ')})`);
+    if (ref) {
+        const actions = (element.actions ?? []).filter((action) => UI_ACTIONS.includes(action));
+        if (actions.length > 0)
+            parts.push(`[${actions.join(', ')}]`);
+        const at = frame && element.bounds ? centreOnImage(frame, element.bounds) : undefined;
+        if (at)
+            parts.push(`@${pointLabel(at)}`);
+    }
+    return neutralizeEnvelopeDelimiter(parts.join(' '));
+}
+/** The centre of a virtual-desktop rectangle on a screenshot, or undefined when it is not on its display. */
+function centreOnImage(frame, bounds) {
+    const x = Math.floor(bounds.x + bounds.width / 2) - frame.display.x;
+    const y = Math.floor(bounds.y + bounds.height / 2) - frame.display.y;
+    if (x < 0 || y < 0 || x >= frame.display.width || y >= frame.display.height)
+        return undefined;
+    return toImagePoint(frame, { x, y });
 }
 /**
  * Factory: given a ComputerUseHost (provided by the consumer — e.g.
- * @namzu/computer-use's SubprocessComputerUseHost), returns a ToolDefinition
- * that routes the discriminated action to the host and maps results back to
- * the SDK's ToolResult shape.
+ * @namzu/computer-use's SubprocessComputerUseHost), returns the
+ * `computer_use` tool.
  *
- * The tool's description reflects the host's frozen capabilities, and any
- * action targeting an unavailable capability is rejected with a clear error
- * rather than hanging or failing silently.
+ * Every screenshot is fitted to the model's image limits and numbered;
+ * every coordinate the model sends is a pixel of one of those screenshots and
+ * is mapped onto the host's display. Actions that change something return a
+ * fresh screenshot after a short settle delay, and `batch` runs several in
+ * one call. The description and schema reflect the host's frozen
+ * capabilities, and any action targeting an unavailable capability is
+ * rejected before the host is touched.
  *
  * @example
  * ```ts
@@ -272,65 +678,672 @@ function unknownOutcomeToToolResult(error) {
  * registry.register(createComputerUseTool(host))
  * ```
  */
-export function createComputerUseTool(host) {
-    return defineTool({
+export function createComputerUseTool(host, options = {}) {
+    const settings = resolveSettings(options);
+    const caps = options.unavailableReason
+        ? {
+            ...host.capabilities,
+            screenshot: false,
+            mouse: false,
+            keyboard: false,
+            cursorPosition: false,
+            clipboard: false,
+            windows: false,
+            regionCapture: false,
+            uiTree: false,
+            unavailableReason: options.unavailableReason,
+        }
+        : host.capabilities;
+    const frames = new ScreenshotFrames();
+    const available = new Set(availableActions(host, caps));
+    // The controls of the latest ui_snapshot, by the ref the model was shown.
+    // Refs count up across snapshots, so a ref from an earlier one is never
+    // silently a different control of the latest: it is simply not here.
+    let uiRefs = new Map();
+    let uiSnapshots = 0;
+    let uiRefCount = 0;
+    const describeUiRef = (ref) => {
+        const entry = uiRefs.get(ref);
+        return entry ? `${uiElementText(entry.element)} (${ref})` : undefined;
+    };
+    const label = (input) => itemLabel(input, describeUiRef);
+    const refusal = (type) => {
+        const required = requiredCapability(type);
+        const why = caps.unavailableReason
+            ? ` ${caps.unavailableReason} Do not retry; tell the user.`
+            : '';
+        if (required !== null && caps[required] !== true)
+            return `computer_use: action "${type}" requires capability "${required}" which is not available on this host (displayServer=${caps.displayServer}).${why}`;
+        if (!available.has(type))
+            return `computer_use: action "${type}" is not supported on this host.${why}`;
+        return null;
+    };
+    /** Capture the display, fit it, and number it. */
+    const capture = async () => {
+        const result = await host.execute({ type: 'screenshot' });
+        if (result.type !== 'screenshot')
+            throw new Error(`computer_use: the host answered a screenshot with "${result.type}"`);
+        const shot = result.result;
+        const image = await fitPng(shot.data, settings.limits);
+        const display = shot.display ?? assumedDisplay(image.sourceWidth, image.sourceHeight);
+        return { frame: frames.record(image, display), image };
+    };
+    const frameFor = (id) => {
+        if (id !== undefined) {
+            const frame = frames.get(id);
+            if (!frame)
+                throw new StepFailure(`there is no screenshot ${id} (the latest is ${frames.latest()?.id ?? 'none'}); take a screenshot and use its coordinates`);
+            return frame;
+        }
+        const latest = frames.latest();
+        if (!latest)
+            throw new StepFailure('no screenshot has been taken yet, so there is nothing for coordinates to refer to; take a screenshot first');
+        return latest;
+    };
+    /**
+     * Keys go to whichever window has focus, and a model that has not looked
+     * does not know which one that is — the terminal running this agent is
+     * the usual answer. When the host can show the screen, look first.
+     */
+    const requireLook = () => {
+        if (available.has('screenshot') && !frames.latest())
+            throw new StepFailure('no screenshot has been taken yet, so nothing shows which window has focus and would receive the keys; take a screenshot first');
+    };
+    const mapPoint = (frame, point, field) => {
+        if (!pointOnImage(frame, point))
+            throw new StepFailure(`${field} ${pointLabel(point)} is outside screenshot ${frame.id}, which is ${frame.imageWidth}x${frame.imageHeight} (x 0–${frame.imageWidth - 1}, y 0–${frame.imageHeight - 1}); coordinates are pixels of the screenshot, not of the screen`);
+        return toDisplayPoint(frame, point);
+    };
+    const runHost = async (action) => {
+        try {
+            const result = await host.execute(action);
+            if (result.type === 'cursor_position') {
+                const frame = frames.latest();
+                if (!frame)
+                    return `at display pixel ${pointLabel(result.point)}`;
+                return `at ${pointLabel(toImagePoint(frame, result.point))} in ${frame.id}`;
+            }
+            return 'done';
+        }
+        catch (error) {
+            if (isOutcomeUnknown(error, action.type))
+                throw new StepFailure(error.message, error);
+            throw new StepFailure(errorText(error));
+        }
+    };
+    const plan = (item, frameId) => {
+        const denied = refusal(item.type);
+        if (denied)
+            throw new StepFailure(denied);
+        const mutating = !READ_ONLY_ACTIONS.has(item.type);
+        const step = (run) => ({
+            item,
+            label: label(item),
+            mutating,
+            run,
+        });
+        switch (item.type) {
+            case 'cursor_position':
+                frameFor(frameId);
+                return step(() => runHost({ type: 'cursor_position' }));
+            case 'mouse_move': {
+                const frame = frameFor(frameId);
+                const to = mapPoint(frame, item.to, 'to');
+                return step(() => runHost({ type: 'mouse_move', to }));
+            }
+            case 'mouse_click': {
+                const buttons = caps.mouseClickButtons;
+                if (buttons && !buttons.includes(item.button))
+                    throw new StepFailure(`computer_use: action "mouse_click" does not support button "${item.button}" on this host.`);
+                const frame = frameFor(frameId);
+                const at = mapPoint(frame, item.at, 'at');
+                return step(() => runHost({ type: 'mouse_click', at, button: item.button }));
+            }
+            case 'mouse_drag': {
+                const buttons = caps.mouseDragButtons;
+                if (buttons && !buttons.includes(item.button))
+                    throw new StepFailure(`computer_use: action "mouse_drag" does not support button "${item.button}" on this host.`);
+                const frame = frameFor(frameId);
+                const from = mapPoint(frame, item.from, 'from');
+                const to = mapPoint(frame, item.to, 'to');
+                return step(() => runHost({ type: 'mouse_drag', from, to, button: item.button }));
+            }
+            case 'scroll': {
+                const frame = frameFor(frameId);
+                const at = mapPoint(frame, item.at, 'at');
+                return step(() => runHost({ type: 'scroll', at, direction: item.direction, amount: item.amount }));
+            }
+            case 'type_text':
+                requireLook();
+                return step(() => runHost({ type: 'type_text', text: item.text }));
+            case 'key':
+                requireLook();
+                return step(() => runHost({ type: 'key', keys: item.keys }));
+            case 'wait':
+                return step(async (signal) => {
+                    await sleep(item.ms, signal);
+                    return 'done';
+                });
+            case 'list_windows':
+                return step(() => listWindows());
+            case 'focus_window':
+                return step(async () => {
+                    let outcome;
+                    try {
+                        outcome = await host.focusWindow(item.window_id);
+                    }
+                    catch (error) {
+                        throw new StepFailure(errorText(error));
+                    }
+                    if (!outcome.ok)
+                        throw new StepFailure(`window ${item.window_id} could not be brought to the front; ${outcome.focusedId ? `window ${outcome.focusedId} is in front` : 'the window in front could not be read'}`);
+                    return 'done';
+                });
+            case 'ui_act': {
+                const entry = uiRefs.get(item.ref);
+                if (!entry)
+                    throw new StepFailure(uiSnapshots === 0
+                        ? `there is no ${item.ref}: no ui_snapshot has been taken yet; take one and use its refs`
+                        : `${item.ref} is not a control of the latest ui_snapshot (u${uiSnapshots}); refs are valid only until the next ui_snapshot, so use the refs it showed`);
+                const offered = entry.element.actions;
+                if (offered && offered.length > 0 && !offered.includes(item.action))
+                    throw new StepFailure(`${describeUiRef(item.ref)} offers ${offered.join(', ')}, not ${item.action}`);
+                if (item.action === 'set_value' && item.value === undefined)
+                    throw new StepFailure('set_value needs value, the text to put in the control');
+                return step(async () => {
+                    let outcome;
+                    try {
+                        outcome = await host.uiAct(entry.hostRef, item.action, item.value);
+                    }
+                    catch (error) {
+                        throw new StepFailure(errorText(error));
+                    }
+                    if (!outcome.ok)
+                        throw new StepFailure(outcome.detail ?? `${item.action} did not take effect on ${describeUiRef(item.ref)}`);
+                    return outcome.detail ? `done (${outcome.detail})` : 'done';
+                });
+            }
+        }
+    };
+    /** Read one window's controls, number the ones the model can act on, and show them. */
+    const uiSnapshot = async (input) => {
+        let snapshot;
+        try {
+            snapshot = await host.uiSnapshot(input.window_id);
+        }
+        catch (error) {
+            throw new StepFailure(errorText(error));
+        }
+        uiSnapshots += 1;
+        const id = `u${uiSnapshots}`;
+        const refs = new Map();
+        const frame = frames.latest();
+        const lines = [];
+        let chars = 0;
+        let shown = 0;
+        let total = 0;
+        let cut = false;
+        const visit = (element, depth) => {
+            total += 1;
+            const actionable = element.ref.length > 0;
+            const children = element.children ?? [];
+            // A nameless control nobody can act on says nothing by itself; its
+            // children still stand, one level up.
+            const silent = !actionable && element.name.trim().length === 0 && element.value === undefined;
+            if (!silent && !cut) {
+                const ref = actionable ? `e${uiRefCount + 1}` : undefined;
+                const line = `${'  '.repeat(depth)}${uiElementLine(element, ref, frame)}`;
+                if (chars + line.length + 1 > MAX_UI_SNAPSHOT_CHARS) {
+                    cut = true;
+                }
+                else {
+                    lines.push(line);
+                    chars += line.length + 1;
+                    shown += 1;
+                    if (ref) {
+                        uiRefCount += 1;
+                        refs.set(ref, { hostRef: element.ref, element });
+                    }
+                }
+            }
+            for (const child of children)
+                visit(child, silent ? depth : depth + 1);
+        };
+        visit(snapshot.root, 0);
+        uiRefs = refs;
+        const window = snapshot.windowId && /^[\w.:-]{1,64}$/.test(snapshot.windowId) ? snapshot.windowId : undefined;
+        const header = [
+            `UI snapshot ${id}${window ? ` of window ${window}` : ''}: ${shown} controls shown, ${refs.size} with a ref you can pass to ui_act. Refs are valid until the next ui_snapshot.`,
+            ...(frame
+                ? [
+                    `@(x, y) is a control's centre on screenshot ${frame.id}, for a click when ui_act cannot reach it.`,
+                ]
+                : []),
+        ];
+        // The window's title and application, when the tree's own root does not
+        // already say them; inside the frame, since an application sets both.
+        const about = [
+            ...(snapshot.app !== undefined ? [`Application ${quotedText(snapshot.app)}`] : []),
+            ...(snapshot.title !== undefined && snapshot.title !== snapshot.root.name
+                ? [`Window title ${quotedText(snapshot.title)}`]
+                : []),
+        ];
+        const body = [...about, ...lines].join('\n');
+        const footer = cut || snapshot.truncated
+            ? [
+                cut
+                    ? `The tree was cut at ${MAX_UI_SNAPSHOT_CHARS} characters after ${shown} of ${total} controls. Read a smaller window, or use a screenshot for the rest.`
+                    : 'The host stopped reading the tree before it ended; controls further down are missing. Use a screenshot for the rest.',
+            ]
+            : [];
+        const text = [
+            ...header,
+            wrapUntrusted({
+                kind: 'desktop-ui',
+                ...(window ? { attributes: { window } } : {}),
+                provenance: "The accessibility tree of a window on the user's desktop, as the host read it. Names and values are whatever the application shows, which can include text anyone wrote.",
+            }, body),
+            ...footer,
+        ].join('\n');
+        return {
+            success: true,
+            output: header[0] ?? '',
+            content: [{ type: 'text', text }],
+            data: {
+                uiSnapshot: {
+                    id,
+                    ...(window ? { windowId: window } : {}),
+                    controls: shown,
+                    refs: refs.size,
+                    truncated: cut || snapshot.truncated === true,
+                },
+            },
+        };
+    };
+    /**
+     * Keys and text go to the window in front, and a terminal there — the
+     * one running this agent, or the user's own — turns a model's typing into
+     * a command line: ENTER runs it or sends it. A session had a WIN+R that
+     * did not open the Run dialog type "notepad" and ENTER into the user's
+     * terminal, and submitted their half-typed message. So with a window list
+     * the tool reads what is in front before typing, and refuses a terminal.
+     * Without one it cannot tell, and the description's advice stands alone.
+     */
+    const refuseTerminalInFront = async () => {
+        if (!available.has('list_windows'))
+            return;
+        let windows;
+        try {
+            windows = await host.listWindows();
+        }
+        catch (error) {
+            throw new StepFailure(`the window in front could not be read, so nothing was typed: ${errorText(error)}`);
+        }
+        const front = windows.find((window) => window.focused);
+        if (front && TERMINAL_APPS.test(front.app))
+            throw new StepFailure(`a terminal is in front (${front.app}, window ${front.id}), so nothing was typed: computer_use never sends keys or text to a terminal. Bring the window you mean to the front (focus_window, or click it) and try again; for a command, use a shell tool instead`);
+    };
+    const listWindows = async () => {
+        let windows;
+        try {
+            windows = await host.listWindows();
+        }
+        catch (error) {
+            throw new StepFailure(errorText(error));
+        }
+        if (windows.length === 0)
+            return 'no windows are open';
+        const frame = frames.latest();
+        const lines = windows.slice(0, MAX_LISTED_WINDOWS).map((window) => {
+            const where = frame
+                ? (() => {
+                    const rect = window.minimized ? null : desktopRectOnImage(frame, window.bounds);
+                    return rect
+                        ? `at ${pointLabel(rect)} ${rect.width}x${rect.height} in ${frame.id}`
+                        : window.minimized
+                            ? 'minimized'
+                            : `not on the display of ${frame.id}`;
+                })()
+                : window.minimized
+                    ? 'minimized'
+                    : '';
+            return [
+                `- ${window.id}`,
+                JSON.stringify(window.title),
+                `${window.app} (pid ${window.pid})`,
+                ...(window.focused ? ['focused'] : []),
+                ...(where ? [where] : []),
+            ].join(' · ');
+        });
+        const more = windows.length > MAX_LISTED_WINDOWS
+            ? [`… and ${windows.length - MAX_LISTED_WINDOWS} more`]
+            : [];
+        // Titles are whatever an application puts there — a web page's title in
+        // a browser window — so the list is framed as material, not direction.
+        return `${windows.length} window${windows.length === 1 ? '' : 's'}, front to back:\n${wrapUntrusted({
+            kind: 'desktop-windows',
+            provenance: "The open windows on the user's desktop, as the host listed them. Titles are whatever each application shows, which can include text anyone wrote.",
+        }, [...lines, ...more].join('\n'))}`;
+    };
+    const describeFrame = (frame) => {
+        const { display } = frame;
+        const scaled = frame.imageWidth !== display.width || frame.imageHeight !== display.height;
+        return scaled
+            ? `Screenshot ${frame.id}: ${frame.imageWidth}x${frame.imageHeight} pixels, showing the ${display.width}x${display.height} display. Send coordinates in this image's pixels (x 0–${frame.imageWidth - 1}, y 0–${frame.imageHeight - 1}); the tool maps them onto the display.`
+            : `Screenshot ${frame.id}: ${frame.imageWidth}x${frame.imageHeight} pixels, the display at full size. Send coordinates in this image's pixels (x 0–${frame.imageWidth - 1}, y 0–${frame.imageHeight - 1}).`;
+    };
+    const frameData = (frame) => ({
+        id: frame.id,
+        width: frame.imageWidth,
+        height: frame.imageHeight,
+        display: frame.display,
+        mimeType: 'image/png',
+        encoding: 'base64',
+    });
+    const pin = (frame) => [
+        {
+            key: 'computer_use.screenshot',
+            text: `computer_use coordinates are pixels of screenshot ${frame.id} (${frame.imageWidth}x${frame.imageHeight}), origin top-left.`,
+        },
+    ];
+    const imageBlock = (image) => ({
+        type: 'image',
+        data: image.data.toString('base64'),
+        mediaType: 'image/png',
+    });
+    // Said once, beside the first screenshot, where it is read: a model that
+    // has just seen the screen reaches for pixels unless told the host can
+    // name the controls.
+    const firstLookHint = (frame) => frame.id === 's1' && available.has('ui_snapshot')
+        ? [
+            'This host can also read a window’s controls: list_windows, then ui_snapshot {window_id}, then ui_act by ref (a batch of them for several buttons) — surer than clicking pixels in an ordinary application.',
+        ]
+        : [];
+    const screenshotResult = (shot) => ({
+        success: true,
+        output: `Screenshot ${shot.frame.id} captured (${shot.frame.imageWidth}x${shot.frame.imageHeight} of the ${shot.frame.display.width}x${shot.frame.display.height} display).`,
+        content: [
+            { type: 'text', text: [describeFrame(shot.frame), ...firstLookHint(shot.frame)].join('\n') },
+            imageBlock(shot.image),
+        ],
+        data: { screenshot: frameData(shot.frame) },
+        workingState: pin(shot.frame),
+    });
+    const zoom = async (input) => {
+        const frame = frameFor(input.screenshot_id);
+        const { region } = input;
+        const onImage = {
+            x: Math.max(0, region.x),
+            y: Math.max(0, region.y),
+            width: Math.min(region.x + region.width, frame.imageWidth) - Math.max(0, region.x),
+            height: Math.min(region.y + region.height, frame.imageHeight) - Math.max(0, region.y),
+        };
+        if (onImage.width <= 0 || onImage.height <= 0)
+            throw new StepFailure(`region ${region.width}x${region.height} at ${pointLabel(region)} is outside screenshot ${frame.id} (${frame.imageWidth}x${frame.imageHeight})`);
+        const rect = toDisplayRect(frame, onImage);
+        let fitted;
+        if (caps.regionCapture === true && typeof host.captureRegion === 'function') {
+            const piece = await host.captureRegion(rect);
+            fitted = await fitPng(piece.data, settings.limits);
+        }
+        else {
+            const result = await host.execute({ type: 'screenshot' });
+            if (result.type !== 'screenshot')
+                throw new Error(`computer_use: the host answered a screenshot with "${result.type}"`);
+            const full = result.result;
+            const size = pngSize(full.data);
+            const display = full.display ?? assumedDisplay(size.width, size.height);
+            if (display.width !== frame.display.width || display.height !== frame.display.height)
+                throw new StepFailure(`the display is now ${display.width}x${display.height}, not what ${frame.id} showed; take a new screenshot`);
+            // The capture's own pixels, in case a host captures at another scale than it reports.
+            const sx = size.width / display.width;
+            const sy = size.height / display.height;
+            const left = Math.floor(rect.x * sx);
+            const top = Math.floor(rect.y * sy);
+            fitted = await cropAndFitPng(full.data, {
+                x: left,
+                y: top,
+                width: Math.max(1, Math.min(size.width, Math.ceil((rect.x + rect.width) * sx)) - left),
+                height: Math.max(1, Math.min(size.height, Math.ceil((rect.y + rect.height) * sy)) - top),
+            }, settings.limits);
+        }
+        const detail = fitted.width / onImage.width;
+        const text = `Zoom of ${onImage.width}x${onImage.height} at ${pointLabel(onImage)} in ${frame.id}, shown at ${fitted.width}x${fitted.height} (${detail.toFixed(1)}x the detail of ${frame.id}). Coordinates for actions still refer to ${frame.id} (${frame.imageWidth}x${frame.imageHeight}), not to this image.`;
+        return {
+            success: true,
+            output: `Zoomed into ${onImage.width}x${onImage.height} at ${pointLabel(onImage)} of ${frame.id} (shown at ${fitted.width}x${fitted.height}).`,
+            content: [{ type: 'text', text }, imageBlock(fitted)],
+            data: {
+                zoom: {
+                    screenshot: frame.id,
+                    region: onImage,
+                    width: fitted.width,
+                    height: fitted.height,
+                    mimeType: 'image/png',
+                    encoding: 'base64',
+                },
+            },
+        };
+    };
+    /** Run actions in order, stop at the first failure, then show the screen once. */
+    const runSteps = async (items, frameId, context, single) => {
+        if (items.length > settings.maxBatchActions)
+            return {
+                success: false,
+                output: '',
+                error: `computer_use: a batch carries at most ${settings.maxBatchActions} actions; got ${items.length}. Nothing was run.`,
+            };
+        // Waits share one budget, so a batch stays well inside the tool's
+        // deadline however its waits are split.
+        const waited = items.reduce((total, item) => total + (item.type === 'wait' ? item.ms : 0), 0);
+        if (waited > settings.maxWaitMs)
+            return {
+                success: false,
+                output: '',
+                error: `computer_use: ${single ? 'wait is' : "a batch's waits add up to"} at most ${settings.maxWaitMs} ms; got ${waited}. Nothing was run.`,
+            };
+        // Plan everything before doing anything: a refusal or a coordinate off
+        // the screenshot in step 5 should not arrive after steps 1–4 changed
+        // the desktop.
+        const planned = [];
+        for (const [index, item] of items.entries()) {
+            try {
+                planned.push(plan(item, frameId));
+            }
+            catch (error) {
+                const message = errorText(error);
+                return {
+                    success: false,
+                    output: '',
+                    error: single
+                        ? message
+                        : `computer_use: action ${index + 1} of ${items.length} (${label(item)}) cannot run: ${message}. Nothing was run.`,
+                };
+            }
+        }
+        const signal = context?.abortSignal;
+        const records = [];
+        let changed = false;
+        let failure;
+        // Whether the window in front was checked since the last step that could
+        // have changed it. Typing does not move focus; everything else may.
+        let frontChecked = false;
+        for (const [index, step] of planned.entries()) {
+            if (signal?.aborted) {
+                failure = { index, error: 'cancelled before it started' };
+                records.push({ label: step.label, status: 'failed', error: failure.error });
+                break;
+            }
+            try {
+                const keyboard = step.item.type === 'type_text' || step.item.type === 'key';
+                if (keyboard && !frontChecked) {
+                    await refuseTerminalInFront();
+                    frontChecked = true;
+                }
+                if (step.item.type !== 'type_text')
+                    frontChecked = false;
+                const note = await step.run(signal);
+                records.push({ label: step.label, status: 'done', note });
+                if (step.mutating)
+                    changed = true;
+            }
+            catch (error) {
+                const unknown = error instanceof StepFailure ? error.unknown : undefined;
+                const message = signal?.aborted ? 'cancelled' : errorText(error);
+                failure = { index, error: message, ...(unknown ? { unknown } : {}) };
+                records.push({
+                    label: step.label,
+                    status: 'failed',
+                    error: message,
+                    ...(unknown ? { unknown } : {}),
+                });
+                if (unknown)
+                    changed = true;
+                break;
+            }
+        }
+        for (const step of planned.slice(records.length))
+            records.push({ label: step.label, status: 'not run' });
+        // The screen after the batch: after anything that may have changed it,
+        // and after a wait, whose whole point is to look again.
+        const looks = settings.screenshotAfterActions &&
+            available.has('screenshot') &&
+            (changed || planned.some((step) => step.item.type === 'wait')) &&
+            !signal?.aborted;
+        let shot;
+        let shotError;
+        if (looks) {
+            try {
+                await sleep(settings.settleMs, signal);
+                shot = await capture();
+            }
+            catch (error) {
+                shotError = signal?.aborted ? 'cancelled' : errorText(error);
+            }
+        }
+        const summary = (record, index) => {
+            const n = single ? '' : `${index + 1}. `;
+            switch (record.status) {
+                case 'done':
+                    return `${n}${record.label}: ${record.note}`;
+                case 'failed':
+                    return `${n}${record.label}: failed — ${record.error}`;
+                case 'not run':
+                    return `${n}${record.label}: not run`;
+            }
+        };
+        const header = single
+            ? []
+            : failure
+                ? [
+                    `Batch stopped at action ${failure.index + 1} of ${planned.length}; ${planned.length - failure.index - 1} not run.`,
+                ]
+                : [`Batch: all ${planned.length} actions done.`];
+        const stepLines = records.map(summary);
+        const shotLines = shot
+            ? [describeFrame(shot.frame)]
+            : shotError
+                ? [`The screenshot after acting failed: ${shotError}. Take one before acting again.`]
+                : [];
+        const unknownLines = failure?.unknown ? [failure.unknown.message] : [];
+        const text = [...header, ...stepLines, ...unknownLines, ...shotLines].join('\n');
+        const output = [
+            ...header,
+            ...stepLines,
+            ...(shot
+                ? [`Screenshot ${shot.frame.id} (${shot.frame.imageWidth}x${shot.frame.imageHeight}).`]
+                : []),
+        ].join('\n');
+        const data = {
+            steps: records.map((record) => record.status === 'failed'
+                ? { label: record.label, status: record.status, error: record.error }
+                : { label: record.label, status: record.status }),
+            ...(shot ? { screenshot: frameData(shot.frame) } : {}),
+            ...(failure?.unknown
+                ? {
+                    code: failure.unknown.code,
+                    action: failure.unknown.action,
+                    outcome: failure.unknown.outcome,
+                    retrySafety: failure.unknown.retrySafety,
+                    timedOut: failure.unknown.timedOut,
+                    exitCode: failure.unknown.exitCode,
+                }
+                : {}),
+        };
+        const content = [{ type: 'text', text }];
+        if (shot)
+            content.push(imageBlock(shot.image));
+        const result = {
+            success: failure === undefined,
+            output,
+            content,
+            data,
+            ...(shot ? { workingState: pin(shot.frame) } : {}),
+        };
+        if (failure)
+            result.error = failure.unknown
+                ? failure.unknown.message
+                : single
+                    ? `computer_use failed: ${failure.error}`
+                    : `Batch stopped at action ${failure.index + 1} of ${planned.length} (${planned[failure.index]?.label}): ${failure.error}`;
+        return result;
+    };
+    // Whether a call sends the screen to the provider: an observation, or an
+    // action that returns a screenshot afterwards. A diagnostic tool that can
+    // reach nothing captures nothing.
+    const observes = available.has('screenshot') || available.has('list_windows') || available.has('ui_snapshot');
+    const capturesScreen = (input) => observes && capturesScreenInput(input, settings.screenshotAfterActions);
+    const tool = defineTool({
         name: COMPUTER_USE_TOOL_NAME,
-        description: buildDescription(host),
+        description: buildDescription(host, caps, settings),
         inputSchema: actionSchema,
-        modelInputSchema: hostModelSchema(host.capabilities),
-        validationErrorHint: 'Action requirements: mouse_move needs "to"; mouse_click needs "at" and "button"; mouse_drag needs "from", "to", and "button"; scroll needs "at", "direction", and positive "amount"; type_text needs "text"; key needs "keys".',
+        modelInputSchema: hostModelSchema(host, caps, settings),
+        validationErrorHint: `Action requirements: ${ALL_ACTIONS.map((action) => ACTION_REQUIREMENTS[action]).join('; ')}. A batch is {"type":"batch","actions":[...]} with at most ${settings.maxBatchActions} actions and no screenshot, zoom, ui_snapshot or batch inside.`,
         category: 'custom',
         permissions: [],
-        readOnly: false,
-        destructive: (input) => DESTRUCTIVE_ACTION_TYPES.has(input.type),
+        readOnly: (input) => isReadOnlyInput(input),
+        destructive: (input) => isDestructiveInput(input),
+        capturesScreen,
         concurrencySafe: false,
         presentCall: (input) => ({
             kind: 'generic',
-            label: actionLabel(input),
+            label: label(input),
             presentation: 'activity',
         }),
-        presentResult: (_input, result) => result.success && result.output.trim().toLowerCase() === 'ok'
+        // Decided from the result alone: a host may present a finished call
+        // without its input (the CLI passes `{}`). An action that only reports
+        // "done" — and the screenshot that followed it, which is for the model —
+        // adds nothing to the call row; observations, batches and failures do.
+        presentResult: (_input, result) => result.success && ACKNOWLEDGEMENT.test(result.output)
             ? { kind: 'generic', label: result.output, visibility: 'hidden' }
             : undefined,
-        async execute(input, _context) {
-            const required = requiredCapability(input.type);
-            if (required !== null && host.capabilities[required] !== true) {
-                return {
-                    success: false,
-                    output: '',
-                    error: `computer_use: action "${input.type}" requires capability "${required}" which is not available on this host (displayServer=${host.capabilities.displayServer}).${host.capabilities.unavailableReason ? ` ${host.capabilities.unavailableReason} Do not retry; tell the user.` : ''}`,
-                };
-            }
-            if (!availableActions(host.capabilities).includes(input.type)) {
-                return {
-                    success: false,
-                    output: '',
-                    error: `computer_use: action "${input.type}" is not supported on this host.`,
-                };
-            }
-            const buttons = input.type === 'mouse_click'
-                ? host.capabilities.mouseClickButtons
-                : input.type === 'mouse_drag'
-                    ? host.capabilities.mouseDragButtons
-                    : undefined;
-            if (buttons && 'button' in input && !buttons.includes(input.button)) {
-                return {
-                    success: false,
-                    output: '',
-                    error: `computer_use: action "${input.type}" does not support button "${input.button}" on this host.`,
-                };
-            }
+        async execute(input, context) {
+            const denied = refusal(input.type);
+            if (denied)
+                return { success: false, output: '', error: denied };
             try {
-                const result = await host.execute(input);
-                return resultToToolResult(result);
+                switch (input.type) {
+                    case 'screenshot':
+                        return screenshotResult(await capture());
+                    case 'zoom':
+                        return await zoom(input);
+                    case 'ui_snapshot':
+                        return await uiSnapshot(input);
+                    case 'batch':
+                        return await runSteps(input.actions, input.screenshot_id, context, false);
+                    default: {
+                        const { screenshot_id: frameId, ...item } = input;
+                        return await runSteps([item], frameId, context, true);
+                    }
+                }
             }
             catch (error) {
-                if (isOutcomeUnknown(error, input.type)) {
-                    return unknownOutcomeToToolResult(error);
-                }
+                if (error instanceof StepFailure)
+                    return { success: false, output: '', error: `computer_use failed: ${error.message}` };
                 throw error;
             }
         },
     });
+    return Object.assign(tool, { describeUiRef });
 }
 //# sourceMappingURL=computer-use.js.map