openmausbot 0.1.86 → 0.1.87

package/dist-server/server/drivers/boxagent.js

@@ -1,5 +1,6 @@
 import { newEventId, newId } from "../contracts.js";
 import { appendNative } from "./native.js";
+import { OMB_ASK_TOOL, answerWithoutPreamble, askQuestionSummary, capAnswerEcho, parseOmbAskQuestions, questionChoices, stripOmbAskBlock, } from "../../shared/ask-question.js";
 const DRIVER_KIND = "boxAgent";
 // overridable so tests and a dev backend can be pointed at instead of the live provider
 const BOX_API = process.env.OMB_BOX_API || "https://ascii.dev/api/box/v1";
@@ -11,7 +12,24 @@ const MODELS = {
         { id: "gpt-5.4", label: "GPT-5.4 (Codex) · on the box" },
     ],
 };
-/** The box runs every harness ascii.dev ships (claude-code, codex, pi, opencode,
+/** The ask contract appended to every prompt. The box harness cannot pause
+ * mid-run, so a question rides the run's final output as a fenced block and
+ * OMB parses it at settle — the turn-held transport. */
+const ASK_PROTOCOL = [
+    "",
+    "## Asking the person a question",
+    "When a decision belongs to the person, end your reply with a fenced block exactly like this:",
+    "",
+    "```omb-ask",
+    '{"questions":[{"question":"Ship the release now?","header":"Release","options":[{"label":"Ship now"},{"label":"Wait for the QA signoff"}]}]}',
+    "```",
+    "",
+    "The block must be the last thing in your reply. You may ask up to 6 questions at once, each with up to 12 options; the person can always answer in their own words. Their answers arrive on your next prompt as `Q:`/`A:` lines — never invent them.",
+].join("\n");
+/** Any fence whose info string names the ask protocol, even when its body
+ * does not parse: the marker for "the model tried to ask and failed". */
+const ASK_FENCE_ANY = /(^|\n)[ \t]{0,3}(`{3,}|~{3,})[ \t]*omb-ask\b/;
+/** The box runs every harness boat.dev ships (claude-code, codex, pi, opencode,
  * prime-agent, kimi). Which one a model id belongs to comes from the public
  * catalog, `GET /api/provider-models` at the API root: an object keyed by
  * harness, each with its `models`. A bot that arrives here from another engine
@@ -35,7 +53,10 @@ const providerFor = (model) => {
 };
 function decodeConfig(raw) {
     const o = (raw ?? {});
-    return { pollMs: typeof o.pollMs === "number" ? o.pollMs : 2500 };
+    return {
+        pollMs: typeof o.pollMs === "number" ? o.pollMs : 2500,
+        ...(typeof o.askTimeoutMs === "number" ? { askTimeoutMs: o.askTimeoutMs } : {}),
+    };
 }
 export const BoxAgentDriver = {
     driverKind: DRIVER_KIND,
@@ -48,6 +69,13 @@ export const BoxAgentDriver = {
         const token = input.environment.BOX_TOKEN ?? process.env.BOX_TOKEN ?? "";
         const listeners = new Set();
         const active = new Map();
+        /** One open ask per thread: the turn-held transport's pending card. The
+         * thread is busy for exactly as long as the ask is open, so a second
+         * block can never race the first. */
+        const heldAsks = new Map();
+        /** Threads whose last run ended with an unparseable omb-ask fence: the
+         * next prompt carries the correction so the ask is never lost silently. */
+        const malformedAsks = new Set();
         const emit = (event) => {
             for (const l of Array.from(listeners))
                 l(event);
@@ -84,46 +112,65 @@ export const BoxAgentDriver = {
                 throw new Error("a turn is already running on this thread");
             const turnId = newId();
             const model = turn.model || MODELS.default;
+            const correction = malformedAsks.delete(threadId)
+                ? "Your previous omb-ask block was malformed or empty, so the person never saw it. Ask again with a valid fenced omb-ask JSON block, or ask in plain words."
+                : "";
             const prompt = [
                 turn.system,
                 "You are working on the assigned cloud computer — use its desktop, Chrome, and shell within the access described above.",
+                ASK_PROTOCOL,
+                ...(correction ? [correction] : []),
                 "",
                 turn.text,
             ]
                 .filter((s) => s !== undefined)
                 .join("\n");
-            if (!catalog)
-                await loadCatalog();
-            const started = await api(`/boxes/${boxId}/prompt`, {
-                method: "POST",
-                body: JSON.stringify({ ...providerFor(model), prompt }),
-            });
-            appendNative(threadId, { dir: "out", source: "box.prompt", msg: { model, prompt, response: started } });
-            // real shape (2026-08): {type:"prompt.queued", promptId, promptRun:{id,…},
-            // id:<box id>} — never fall back to the bare id, it's the box's
-            const promptId = started?.promptRun?.id ?? started?.prompt?.id ?? started?.promptId ?? null;
+            const postPrompt = async (promptText) => {
+                if (!catalog)
+                    await loadCatalog();
+                const started = await api(`/boxes/${boxId}/prompt`, {
+                    method: "POST",
+                    body: JSON.stringify({ ...providerFor(model), prompt: promptText }),
+                });
+                appendNative(threadId, { dir: "out", source: "box.prompt", msg: { model, prompt: promptText, response: started } });
+                // real shape (2026-08): {type:"prompt.queued", promptId, promptRun:{id,…},
+                // id:<box id>} — never fall back to the bare id, it's the box's
+                return started?.promptRun?.id ?? started?.prompt?.id ?? started?.promptId ?? null;
+            };
+            const promptId = await postPrompt(prompt);
             let cancelled = false;
             active.set(threadId, {
                 turnId,
                 boxId,
                 cancel: () => {
                     cancelled = true;
+                    heldAsks.get(threadId)?.settle(null, "system");
                     void api(`/boxes/${boxId}/interrupt`, { method: "POST" }).catch(() => { });
                 },
             });
             emit({ ...base(threadId, turnId), type: "turn.started" });
             emit({ ...base(threadId, turnId), type: "session.started", sessionId: promptId, model });
-            // poll events + run status until the prompt settles
+            // poll events + run status until the prompt settles — and, when the
+            // run ends on an omb-ask block, until the person answers it: the OMB
+            // turn is the unit the whole server already understands (busy thread,
+            // waiting-on-you, routines waiting), so it stays open across the ask
+            // and the continuation prompt continues it rather than starting a
+            // driver-initiated turn nobody accounts for.
             (async () => {
+                // Event ids seen this TURN. The events stream is the whole
+                // conversation, so a continuation run must not re-ingest history;
+                // `taskId` normally filters it, but ids survive even shape drift.
                 const seen = new Set();
-                const startedAt = Date.now();
                 let lastText = "";
                 let pendingText = "";
                 /** Why the box could not answer (login expired, model refused, …). */
                 let problem = null;
-                /** Emit unflushed deltas as assistant_text and reset pendingText. */
+                /** Emit unflushed deltas as assistant_text and reset pendingText.
+                 * The omb-ask block is protocol, not prose: it streamed raw (the
+                 * card is its readable form), and the settled message shows the
+                 * words around it, never the JSON. */
                 const flushAssistantText = () => {
-                    const text = pendingText;
+                    const text = stripOmbAskBlock(pendingText);
                     pendingText = "";
                     if (!text.trim())
                         return;
@@ -138,7 +185,88 @@ export const BoxAgentDriver = {
                     pendingText += delta;
                     emit({ ...base(threadId, turnId), type: "content.delta", streamKind: "assistant_text", delta });
                 };
-                try {
+                /** The run has settled: flush its prose, then hold the OMB turn open
+                 * on an omb-ask block (if any) until the person answers or the ask
+                 * times out. An answer chains the continuation run under this same
+                 * turn; a deny or timeout ends it. */
+                const finishRun = async (ok, stopReason) => {
+                    flushAssistantText();
+                    // An interrupt that lands between ask text streaming and settle
+                    // finds no heldAsks entry to settle; without this check the run
+                    // would register a fresh ask for a dead turn and hold stop pending
+                    // for the ask timeout.
+                    if (cancelled) {
+                        return { ok: false, stopReason: "interrupted" };
+                    }
+                    // A remotely interrupted or failed run must stay stopped, even
+                    // if its partial output already contains a complete question.
+                    if (!ok)
+                        return { ok, stopReason };
+                    const questions = parseOmbAskQuestions(lastText);
+                    if (!questions) {
+                        // A fence that did not parse is a question the person never saw.
+                        // Say so on the next prompt instead of losing it silently. (An
+                        // over-cap block does not land here: the parser caps it at six
+                        // questions and still shows the card.)
+                        if (ASK_FENCE_ANY.test(lastText))
+                            malformedAsks.add(threadId);
+                        return { ok: ok && !cancelled, stopReason: cancelled ? "interrupted" : stopReason };
+                    }
+                    const requestId = newId();
+                    let settleHeld;
+                    const held = new Promise((resolve) => {
+                        const timer = setTimeout(() => settleHeld(null, "timeout"), config.askTimeoutMs ?? 15 * 60_000);
+                        timer.unref?.();
+                        settleHeld = (reply, source) => {
+                            heldAsks.delete(threadId);
+                            clearTimeout(timer);
+                            resolve([reply, source]);
+                        };
+                    });
+                    // register before emitting — an answer can race the emit
+                    heldAsks.set(threadId, { requestId, settle: settleHeld });
+                    const choices = questionChoices(questions);
+                    emit({
+                        ...base(threadId, turnId),
+                        requestId,
+                        type: "request.opened",
+                        requestType: "question",
+                        tool: OMB_ASK_TOOL,
+                        summary: askQuestionSummary(questions),
+                        questions,
+                        ...(choices?.length ? { choices } : {}),
+                        origin: "output",
+                    });
+                    const [reply, source] = await held;
+                    emit({ ...base(threadId, turnId), requestId, type: "request.resolved", behavior: reply !== null ? "answer" : "deny", source });
+                    if (reply === null || cancelled) {
+                        return { ok: ok && !cancelled, stopReason: cancelled ? "interrupted" : stopReason };
+                    }
+                    // The turn is still open, so this prompt continues it: same turnId,
+                    // same accounting, and the answer reaches the box the way the ask
+                    // contract promised — Q:/A: blocks, capped for echo.
+                    const continuation = [
+                        capAnswerEcho(answerWithoutPreamble(reply)),
+                        "",
+                        "The person answered the omb-ask questions above (Q:/A:). Continue the task with their answers; end with another omb-ask block only if you truly need more.",
+                    ].join("\n");
+                    const nextPromptId = await postPrompt(continuation);
+                    // Stop can land while the continuation POST is in flight: the
+                    // interrupt inside cancel() then hits a box with no active run,
+                    // and the continuation would start after it. Interrupt the run
+                    // that just started before ending the turn.
+                    if (cancelled) {
+                        void api(`/boxes/${boxId}/interrupt`, { method: "POST" }).catch(() => { });
+                        return { ok: false, stopReason: "interrupted" };
+                    }
+                    return await settleRun(nextPromptId);
+                };
+                /** Poll one box run to its settle. */
+                const settleRun = async (runPromptId) => {
+                    const startedAt = Date.now(); // one 30-min ceiling per box run, ask chains included
+                    lastText = "";
+                    pendingText = "";
+                    problem = null;
                     for (;;) {
                         if (cancelled)
                             break;
@@ -149,7 +277,7 @@ export const BoxAgentDriver = {
                             // The stream is the whole conversation from its start; `taskId`
                             // names the prompt run an event belongs to. Earlier turns are
                             // history, not this answer.
-                            if (promptId && ev.taskId && String(ev.taskId) !== promptId)
+                            if (runPromptId && ev.taskId && String(ev.taskId) !== runPromptId)
                                 continue;
                             const id = String(ev.id ?? ev.eventId ?? JSON.stringify(ev).slice(0, 120));
                             if (seen.has(id))
@@ -184,16 +312,13 @@ export const BoxAgentDriver = {
                             // shape-drift backstop: without a promptId the status poll
                             // below can never see a terminal state, so settle off the
                             // events themselves instead of hanging to the 30-min ceiling
-                            if (!promptId && /complete|finish|done|success|fail|error/i.test(kind)) {
-                                active.delete(threadId);
-                                flushAssistantText();
+                            if (!runPromptId && /complete|finish|done|success|fail|error/i.test(kind)) {
                                 const failed = /fail|error/i.test(kind);
-                                emit({ ...base(threadId, turnId), type: "turn.completed", ok: !failed, stopReason: failed ? kind : null, cost: null });
-                                return;
+                                return await finishRun(!failed, failed ? kind : null);
                             }
                         }
-                        if (promptId) {
-                            const status = await api(`/boxes/${boxId}/prompts/${promptId}`).catch(() => null);
+                        if (runPromptId) {
+                            const status = await api(`/boxes/${boxId}/prompts/${runPromptId}`).catch(() => null);
                             appendNative(threadId, { dir: "in", source: "box.prompt.status", msg: status });
                             // real shape (2026-08): {promptRun:{status:"finished",…}} —
                             // flat fallbacks kept for drift
@@ -211,19 +336,13 @@ export const BoxAgentDriver = {
                                         throw new Error(problem);
                                     pendingText = "(finished)";
                                 }
-                                flushAssistantText();
-                                active.delete(threadId);
-                                emit({ ...base(threadId, turnId), type: "turn.completed", ok: true, stopReason: null, cost: null });
-                                return;
+                                return await finishRun(true, null);
                             }
                             if (/failed|error|cancelled|interrupted/i.test(state)) {
                                 const runError = [run?.error, run?.failureReason, run?.message].find((v) => typeof v === "string" && v.trim());
                                 if (problem || runError || /failed|error/i.test(state))
                                     throw new Error(problem ?? runError ?? `the box run ${state}`);
-                                flushAssistantText();
-                                active.delete(threadId);
-                                emit({ ...base(threadId, turnId), type: "turn.completed", ok: false, stopReason: state, cost: null });
-                                return;
+                                return await finishRun(false, state);
                             }
                         }
                         if (Date.now() - startedAt > 30 * 60_000) {
@@ -231,12 +350,16 @@ export const BoxAgentDriver = {
                         }
                     }
                     // cancelled
-                    flushAssistantText();
+                    return await finishRun(false, "interrupted");
+                };
+                try {
+                    const outcome = await settleRun(promptId);
                     active.delete(threadId);
-                    emit({ ...base(threadId, turnId), type: "turn.completed", ok: false, stopReason: "interrupted", cost: null });
+                    emit({ ...base(threadId, turnId), type: "turn.completed", ok: outcome.ok, stopReason: outcome.stopReason, cost: null });
                 }
                 catch (e) {
                     flushAssistantText();
+                    heldAsks.delete(threadId);
                     active.delete(threadId);
                     emit({ ...base(threadId, turnId), type: "runtime.error", message: e.message });
                     emit({ ...base(threadId, turnId), type: "turn.completed", ok: false, stopReason: "error", cost: null });
@@ -268,7 +391,22 @@ export const BoxAgentDriver = {
                 capabilities: { sessionModelSwitch: "in-session" },
                 sendTurn,
                 interruptTurn: async (threadId) => active.get(threadId)?.cancel(),
-                respondToRequest: async () => "unavailable", // this engine has no asks to answer
+                respondToRequest: async (threadId, requestId, decision) => {
+                    const held = heldAsks.get(threadId);
+                    if (!held || held.requestId !== requestId)
+                        return "unavailable"; // settled, timed out, or restarted away
+                    const reply = decision.message?.trim();
+                    if (decision.behavior === "answer" && reply) {
+                        held.settle(decision.message, "user");
+                        return "answered";
+                    }
+                    if (decision.behavior === "deny") {
+                        held.settle(null, "user");
+                        return "rejected";
+                    }
+                    // "allow" is not an answer to a question, and blank text is not either
+                    return "unavailable";
+                },
                 hasSession: (threadId) => active.has(threadId),
                 stopAll: async () => {
                     for (const { cancel } of active.values())

package/dist-server/server/drivers/builtIn.js

@@ -14,6 +14,7 @@ import { CustomAcpDriver } from "./acp/custom.js";
 import { HermesAgentDriver } from "./acp/hermes.js";
 import { OpenAICompatDriver } from "./openai-compat.js";
 import { PiDriver } from "./pi.js";
+import { MistralDriver } from "./mistral.js";
 import { MinimaxDriver } from "./minimax.js";
 export const BUILT_IN_DRIVERS = [
     GrokDriver,
@@ -33,4 +34,5 @@ export const BUILT_IN_DRIVERS = [
     AntigravityDriver,
     BoxAgentDriver,
     MinimaxDriver,
+    MistralDriver,
 ];

package/dist-server/server/drivers/chat-box-tools.js

@@ -0,0 +1,90 @@
+import { isolatedRemoteCommand, runCommand, screenshotBox } from "../box.js";
+import { createControlClient, CONTROL_REFUSAL_PLAIN } from "../control-client.js";
+const coordinate = { type: "integer", minimum: 0, maximum: 32767 };
+const tool = (name, description, properties = {}, required = []) => ({ name, description, inputSchema: { type: "object", properties, required, additionalProperties: false } });
+const TOOLS = [
+    tool("screenshot", "See the assigned cloud desktop at native pixel resolution."),
+    tool("get_screen_size", "Get the cloud desktop width and height."),
+    tool("click", "Click a point on the cloud desktop.", { x: coordinate, y: coordinate, button: { type: "string", enum: ["left", "middle", "right"] }, count: { type: "integer", minimum: 1, maximum: 3 } }, ["x", "y"]),
+    tool("move", "Move the cloud mouse pointer.", { x: coordinate, y: coordinate }, ["x", "y"]),
+    tool("drag", "Drag on the cloud desktop.", { x: coordinate, y: coordinate, to_x: coordinate, to_y: coordinate }, ["x", "y", "to_x", "to_y"]),
+    tool("type_text", "Type Unicode text into the focused cloud application.", { text: { type: "string", maxLength: 4000 } }, ["text"]),
+    tool("key_press", "Press an X11 key or shortcut.", { key: { type: "string", pattern: "^[A-Za-z0-9_+]+$", minLength: 1, maxLength: 100 } }, ["key"]),
+    tool("scroll", "Scroll at a point on the cloud desktop.", { x: coordinate, y: coordinate, direction: { type: "string", enum: ["up", "down", "left", "right"] }, amount: { type: "integer", minimum: 1, maximum: 30 } }, ["x", "y", "direction"]),
+    tool("open_url", "Request opening an HTTP or HTTPS URL in the cloud desktop browser; inspect the screen to confirm it loaded.", { url: { type: "string", pattern: "^https?://", maxLength: 2000 } }, ["url"]),
+    tool("exec", "Run a shell command on the assigned cloud computer.", { command: { type: "string", minLength: 1, maxLength: 4000 } }, ["command"]),
+];
+const quote = (value) => "'" + value.replace(/'/g, "'\\''") + "'";
+const buttons = { left: 1, middle: 2, right: 3 };
+const scrollButtons = { up: 4, down: 5, left: 6, right: 7 };
+/** Uses only the leased descriptor supplied by the harness. Schema validation
+ * and user approval happen in the shared chat tool executor before this call. */
+export class ChatBoxClient {
+    closed = false;
+    control;
+    descriptor;
+    constructor(descriptor) {
+        this.descriptor = descriptor;
+        if (!descriptor.control?.url || !descriptor.control.token)
+            throw new Error("Cloud computer control gate missing");
+        this.control = createControlClient(descriptor.control);
+    }
+    async tools() { return TOOLS; }
+    async close() { this.closed = true; }
+    async call(_method, params, signal) {
+        signal.throwIfAborted();
+        if (this.closed)
+            throw new Error("Cloud computer session closed");
+        const state = await this.control.state(true);
+        signal.throwIfAborted();
+        if (this.closed)
+            throw new Error("Cloud computer session closed");
+        if (state.held || state.blockedReason)
+            return { isError: true, content: [{ type: "text", text: state.blockedReason || CONTROL_REFUSAL_PLAIN }] };
+        const { name, arguments: a } = params;
+        const config = { box: { token: this.descriptor.token } };
+        const id = this.descriptor.boxId;
+        if (name === "screenshot") {
+            const shot = await screenshotBox(config, "", id, { signal, nativeSize: true });
+            signal.throwIfAborted();
+            return { content: [{ type: "image", mimeType: "image/jpeg", data: shot.png }] };
+        }
+        const mouse = `xdotool mousemove --sync ${a.x} ${a.y}`;
+        let command;
+        switch (name) {
+            case "get_screen_size":
+                command = "xdotool getdisplaygeometry";
+                break;
+            case "click":
+                command = `${mouse} click --repeat ${a.count ?? 1} --delay 100 ${buttons[String(a.button ?? "left")]}`;
+                break;
+            case "move":
+                command = mouse;
+                break;
+            case "drag":
+                command = `${mouse} mousedown 1 mousemove --sync ${a.to_x} ${a.to_y} mouseup 1`;
+                break;
+            case "type_text":
+                command = `printf %s ${quote(Buffer.from(String(a.text)).toString("base64"))} | base64 -d | xclip -selection clipboard && xdotool key --clearmodifiers ctrl+v`;
+                break;
+            case "key_press":
+                command = `xdotool key --clearmodifiers ${quote(String(a.key))}`;
+                break;
+            case "scroll":
+                command = `${mouse} click --repeat ${a.amount ?? 3} --delay 80 ${scrollButtons[String(a.direction)]}`;
+                break;
+            case "open_url":
+                command = `nohup xdg-open ${quote(String(a.url))} >/dev/null 2>&1 </dev/null &`;
+                break;
+            case "exec":
+                command = String(a.command);
+                break;
+            default: throw new Error("Unknown cloud computer tool");
+        }
+        const result = await runCommand(config, id, isolatedRemoteCommand(command), { signal });
+        signal.throwIfAborted();
+        if (name === "open_url" && result.ok)
+            return { content: [{ type: "text", text: "Browser launch requested. Page loading is not confirmed; inspect the screen before continuing." }] };
+        return { isError: !result.ok, content: [{ type: "text", text: JSON.stringify(result) }] };
+    }
+}

package/dist-server/server/drivers/chat-images.js

@@ -0,0 +1,54 @@
+import { readFileSync, statSync } from "node:fs";
+const IMAGE_BYTES = 20 * 1024 * 1024;
+const MIME = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
+export const CHAT_IMAGE_BUDGET = 32 * 1024 * 1024;
+/** Bound retained base64 before JSON serialization or another model call. */
+export function chatImageBudget() {
+    let used = 0;
+    return (parts) => {
+        if (!Array.isArray(parts))
+            return;
+        const added = parts.reduce((total, part) => total + (part.type === "image_url" ? part.image_url.url.length : 0), 0);
+        if (used + added > CHAT_IMAGE_BUDGET)
+            throw new Error("Turn image budget exceeded (32 MiB encoded); screenshot cannot be retained. An operation may already have taken effect; inspect its state before retrying.");
+        used += added;
+    };
+}
+export function assertImageTransport(url) {
+    const endpoint = new URL(url);
+    const loopback = endpoint.hostname === "localhost" || endpoint.hostname === "[::1]" || /^127(?:\.\d{1,3}){3}$/.test(endpoint.hostname);
+    if (endpoint.protocol !== "https:" && !(endpoint.protocol === "http:" && loopback)) {
+        throw new Error("Images and computer tools require HTTPS for remote API endpoints; HTTP is supported only on loopback.");
+    }
+}
+export function chatImage(item) {
+    const { mimeType, data } = item;
+    if (typeof mimeType !== "string" || !MIME.has(mimeType) || typeof data !== "string" ||
+        !data.length || data.length > Math.ceil(IMAGE_BYTES / 3) * 4 || data.length % 4 ||
+        !/^[A-Za-z0-9+/]*={0,2}$/.test(data)) {
+        throw new Error("Invalid or oversized MCP image");
+    }
+    const bytes = Buffer.from(data, "base64");
+    const matches = mimeType === "image/png" ? bytes.length >= 33 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex")) && bytes.toString("ascii", 12, 16) === "IHDR"
+        : mimeType === "image/jpeg" ? bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff
+            : mimeType === "image/gif" ? bytes.length >= 13 && ["GIF87a", "GIF89a"].includes(bytes.toString("ascii", 0, 6))
+                : bytes.length >= 16 && bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WEBP";
+    if (bytes.length > IMAGE_BYTES || bytes.toString("base64") !== data || !matches)
+        throw new Error("Image bytes do not match the declared MIME type");
+    return { type: "image_url", image_url: { url: `data:${mimeType};base64,${data}` } };
+}
+export function chatUserContent(turn) {
+    if (!turn.images?.length)
+        return turn.text;
+    const budget = chatImageBudget();
+    return [{ type: "text", text: turn.text }, ...turn.images.map(image => {
+            if (statSync(image.path).size > IMAGE_BYTES)
+                throw new Error("Image exceeds 20 MB");
+            const part = chatImage({ mimeType: image.mime, data: readFileSync(image.path).toString("base64") });
+            budget([part]);
+            return part;
+        })];
+}
+export function chatToolImages(callId, images) {
+    return images?.length ? [{ type: "text", text: `Screenshot result from tool call ${callId}:` }, ...images] : [];
+}

package/dist-server/server/drivers/chat-mcp-tools.js

@@ -6,6 +6,8 @@ import formats from "ajv-formats";
 import { stripControlPlaneEnv } from "../config.js";
 import { augmentedPath } from "../env-path.js";
 import { killCliTree, spawnCli } from "../procs.js";
+import { chatImage } from "./chat-images.js";
+import { ChatBoxClient } from "./chat-box-tools.js";
 /** The transport cannot safely continue this turn. A dispatched operation may
  * already have taken effect, so callers must not retry it through a new round. */
 export class ChatToolSessionError extends Error {
@@ -32,6 +34,7 @@ export function chatMcpEnvironment(serverEnv, source = process.env) {
     return { ...env, ...serverEnv };
 }
 class ChatMcpClient {
+    frameBytes;
     child;
     buffer = "";
     nextId = 1;
@@ -39,7 +42,8 @@ class ChatMcpClient {
     closed = false;
     closing;
     pending = new Map();
-    constructor(server) {
+    constructor(server, computerUse = false) {
+        this.frameBytes = computerUse ? 32 * 1024 * 1024 : FRAME_BYTES;
         try {
             // The desktop shell inherits Finder's bare PATH, where `npx`-style
             // servers cannot find `node` and exit at once. Widen it the way the
@@ -89,7 +93,7 @@ class ChatMcpClient {
         if (this.closed || !this.child.stdin.writable || this.child.stdin.destroyed)
             throw new Error("MCP session closed");
         const encoded = JSON.stringify(frame) + "\n";
-        if (Buffer.byteLength(encoded) > FRAME_BYTES)
+        if (Buffer.byteLength(encoded) > this.frameBytes)
             throw new Error("MCP request exceeds the frame limit");
         this.child.stdin.write(encoded);
     }
@@ -103,7 +107,7 @@ class ChatMcpClient {
             this.buffer = this.buffer.slice(newline + 1);
             if (++this.frames > MAX_FRAMES)
                 return this.fail(new Error("MCP session exceeded the response frame count limit"));
-            if (Buffer.byteLength(line) > FRAME_BYTES)
+            if (Buffer.byteLength(line) > this.frameBytes)
                 return this.fail(new Error("MCP response exceeds the frame limit"));
             if (!line.trim())
                 continue;
@@ -140,7 +144,7 @@ class ChatMcpClient {
             else
                 entry.resolve(message.result);
         }
-        if (Buffer.byteLength(this.buffer) > FRAME_BYTES)
+        if (Buffer.byteLength(this.buffer) > this.frameBytes)
             this.fail(new Error("MCP response exceeds the frame limit"));
     }
     async call(method, params, signal, timeout) {
@@ -244,6 +248,8 @@ function compileSchema(schema) {
     // ajv-formats is CommonJS and exports the plugin as both module.exports
     // and .default; the latter also matches its NodeNext declaration.
     formats.default(compiler);
+    compiler.addFormat("uint32", { type: "number", validate: value => Number.isInteger(value) && value >= 0 && value <= 4294967295 });
+    compiler.addFormat("uint64", { type: "number", validate: value => Number.isSafeInteger(value) && value >= 0 });
     try {
         return compiler.compile(schema);
     }
@@ -260,8 +266,14 @@ function boundedText(value) {
         end -= 1;
     return `${bytes.subarray(0, end).toString()}\n[MCP result truncated at 50KB; request less output.]`;
 }
-export async function mountChatTools(integrations, signal) {
+export async function mountChatTools(integrations, signal, computerUse = false) {
     const servers = [];
+    if (computerUse && integrations?.computer)
+        servers.push(["computer", integrations.computer]);
+    if (computerUse && integrations?.localComputer)
+        servers.push(["computer", integrations.localComputer]);
+    if (computerUse && integrations?.browser)
+        servers.push(["browser", integrations.browser]);
     if (integrations?.agents)
         servers.push(["agents", integrations.agents]);
     if (integrations?.composio)
@@ -300,7 +312,9 @@ export async function mountChatTools(integrations, signal) {
         const mounts = await Promise.allSettled(servers.map(async ([name, descriptor]) => {
             if (signal.aborted || closed)
                 throw aborted();
-            const client = new ChatMcpClient(descriptor);
+            // Every mounted MCP server can return images when the caller enables
+            // image delivery, including custom servers. Text stays bounded below.
+            const client = "boxId" in descriptor ? new ChatBoxClient(descriptor) : new ChatMcpClient(descriptor, computerUse);
             clients.push(client);
             return { name, client, tools: await client.tools(signal) };
         }));
@@ -320,6 +334,18 @@ export async function mountChatTools(integrations, signal) {
                 if (Buffer.byteLength(JSON.stringify(tool.inputSchema)) > SCHEMA_BYTES)
                     throw new Error("MCP tool schema exceeds the 64KB limit");
                 const schema = compileSchema(tool.inputSchema);
+                const parameters = { ...tool.inputSchema };
+                const constraints = {};
+                if (computerUse) {
+                    for (const key of ["anyOf", "oneOf", "allOf", "not"]) {
+                        if (key in parameters) {
+                            constraints[key] = parameters[key];
+                            delete parameters[key];
+                        }
+                    }
+                }
+                const description = (typeof tool.description === "string" ? tool.description : "Configured MCP tool") +
+                    (Object.keys(constraints).length ? " Additional argument constraints (validated before execution): " + JSON.stringify(constraints) : "");
                 const base = `${server}_${tool.name}`.toLowerCase().replace(/[^a-z0-9_]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 64) || "mcp_tool";
                 let name = base;
                 for (let index = 2; registered.has(name); index += 1) {
@@ -327,7 +353,7 @@ export async function mountChatTools(integrations, signal) {
                     name = base.slice(0, 64 - suffix.length) + suffix;
                 }
                 registered.set(name, { client, name: tool.name, schema });
-                definitions.push({ type: "function", function: { name, description: typeof tool.description === "string" ? tool.description : "Configured MCP tool", parameters: tool.inputSchema } });
+                definitions.push({ type: "function", function: { name, description, parameters } });
                 if (Buffer.byteLength(JSON.stringify(definitions)) > CATALOG_BYTES)
                     throw new Error("MCP tool catalog exceeds the 1MB limit");
             }
@@ -364,12 +390,15 @@ export async function mountChatTools(integrations, signal) {
                 if (!object(result) || !Array.isArray(result.content) || (result.isError !== undefined && typeof result.isError !== "boolean"))
                     throw new Error("MCP tool returned an invalid result; execution outcome may be uncertain");
                 const parts = [];
+                const images = [];
                 let unsupported = 0;
                 for (const item of result.content) {
                     if (!object(item) || typeof item.type !== "string" || (item.type === "text" && typeof item.text !== "string"))
                         throw new Error("MCP tool returned invalid content; execution outcome may be uncertain");
                     if (item.type === "text")
                         parts.push(item.text);
+                    else if (computerUse && item.type === "image")
+                        images.push(chatImage(item));
                     else
                         unsupported += 1;
                 }
@@ -377,7 +406,8 @@ export async function mountChatTools(integrations, signal) {
                     parts.push(JSON.stringify(result.structuredContent));
                 if (unsupported)
                     parts.unshift(`[${unsupported} unsupported MCP content item(s) omitted. The operation may have taken effect, but its full result cannot be represented; inspect its state before retrying.]`);
-                return { text: boundedText(parts.join("\n") || "(empty result)"), ok: result.isError !== true && unsupported === 0 };
+                return { text: boundedText(parts.join("\n") || (images.length ? "Screenshot captured." : "(empty result)")), ok: result.isError !== true && unsupported === 0,
+                    ...(images.length ? { images } : {}) };
             }
             catch (error) {
                 await close();