openmausbot 0.1.86 → 0.1.88

package/dist-server/server/drivers/openai-chat.js

@@ -1,7 +1,10 @@
 import { newEventId, newId } from "../contracts.js";
+import { ASK_USER_TOOL, ASK_USER_TOOL_DEFINITION, askQuestionSummary, parseAskQuestions, questionChoices } from "../../shared/ask-question.js";
 import { redactSecretsInText } from "../redact.js";
 import { toolDetailPreview } from "../tool-summary.js";
 import { ChatToolSessionError, mountChatTools } from "./chat-mcp-tools.js";
+import { assertImageTransport, chatImageBudget, chatToolImages, chatUserContent } from "./chat-images.js";
+import { promptHalves, volatileContextNote, withContextNote } from "./prompt-split.js";
 import { createChatToolApproval } from "./chat-tool-approval.js";
 import { ChatProtocolError, ChatReasoningDetails, ChatToolCalls, MAX_CHAT_TOOL_CALLS, object } from "./openai-chat-protocol.js";
 import { appendNative } from "./native.js";
@@ -25,6 +28,26 @@ const usageFrom = (usage) => usage
     ? { input: usage.prompt_tokens ?? 0, output: usage.completion_tokens ?? 0 }
     : null;
 const asError = (value) => value instanceof Error ? value : new Error(String(value));
+class UnsupportedChatToolsError extends Error {
+}
+function rejectsToolsParameter(status, body) {
+    if (status !== 400 && status !== 422)
+        return false;
+    let envelope;
+    try {
+        envelope = object(JSON.parse(body));
+    }
+    catch {
+        return false;
+    }
+    const error = object(envelope?.error);
+    if (error?.param === "tools" && error.code === "unsupported_parameter")
+        return true;
+    const message = error?.message ?? envelope?.error;
+    return typeof message === "string" && (/\bdoes not support (?:tools|tool calling|function calling)(?:[.!]?$|[.!]?\s)/i.test(message) ||
+        /\b(?:tools|tool calling|function calling) (?:is|are) not supported\b/i.test(message) ||
+        /^(?:unsupported|unknown|unrecognized) (?:parameter|field):?\s*['"]?tools['"]?[.!]?$/i.test(message.trim()));
+}
 /** Shared runtime for the three providers that speak OpenAI chat completions. */
 export function createOpenAIChatRuntime(options) {
     const { input } = options;
@@ -59,6 +82,7 @@ export function createOpenAIChatRuntime(options) {
                 : timeoutController.signal;
             const response = await fetch(`${options.apiUrl}/chat/completions`, {
                 method: "POST",
+                ...(messages.some(message => Array.isArray(message.content) && message.content.some(part => part.type === "image_url")) ? { redirect: "error" } : {}),
                 headers: { authorization: `Bearer ${options.apiKey}`, "content-type": "application/json" },
                 body: JSON.stringify({
                     ...options.requestBody(model, messages, stream),
@@ -68,7 +92,10 @@ export function createOpenAIChatRuntime(options) {
             });
             if (!response.ok) {
                 const body = await response.text().catch(() => "");
-                throw new Error(`${options.httpErrorLabel} HTTP ${response.status}${body ? `: ${body.slice(0, 200)}` : ""}`);
+                const message = `${options.httpErrorLabel} HTTP ${response.status}${body ? `: ${body.slice(0, 200)}` : ""}`;
+                if (rejectsToolsParameter(response.status, body))
+                    throw new UnsupportedChatToolsError(message);
+                throw new Error(message);
             }
             if (!stream || response.headers.get("content-type")?.includes("application/json")) {
                 const json = await response.json();
@@ -89,7 +116,7 @@ export function createOpenAIChatRuntime(options) {
                 const finishReason = json.choices?.[0]?.finish_reason ?? null;
                 activeSignal.throwIfAborted();
                 return {
-                    text: typeof message?.content === "string" ? message.content : "",
+                    text: options.contentText ? options.contentText(message?.content) : typeof message?.content === "string" ? message.content : "",
                     reasoning: options.reasoning && typeof reasoning === "string"
                         ? reasoning
                         : "",
@@ -155,7 +182,7 @@ export function createOpenAIChatRuntime(options) {
                 const reasoningDelta = options.reasoning && typeof reasoningPart === "string"
                     ? reasoningPart
                     : "";
-                const contentDelta = typeof delta?.content === "string" ? delta.content : "";
+                const contentDelta = options.contentText ? options.contentText(delta?.content) : typeof delta?.content === "string" ? delta.content : "";
                 if (reasoningDelta) {
                     reasoning += reasoningDelta;
                     onDelta?.(reasoningDelta, "reasoning_text");
@@ -220,24 +247,44 @@ export function createOpenAIChatRuntime(options) {
                 clearTimeout(idleTimer);
         }
     };
-    const messagesFor = (turn) => [
-        ...(turn.system ? [{ role: "system", content: turn.system }] : []),
-        ...(turn.transcript ?? []).map((message) => ({
-            role: message.role,
-            content: message.text,
-        })),
-        { role: "user", content: turn.text },
-    ];
+    const messagesFor = (turn) => {
+        // The system message is the head of the resent prefix, so only the
+        // stable half belongs there: a volatile edit must not re-price the
+        // tools, instructions and transcript the provider already cached.
+        // The volatile half rides the newest user message instead, every
+        // turn. Unlike a CLI session, this request is rebuilt from the stored
+        // transcript, which never contains the delivered notes, so tracking a
+        // digest and delivering only on change would leave the model without
+        // its memory on unchanged turns. The newest message is fresh input
+        // on every request anyway.
+        const halves = promptHalves(turn);
+        const note = halves.stable !== null ? volatileContextNote(halves.volatile, false) : "";
+        const userTurn = note ? { ...turn, text: withContextNote(note, turn.text) } : turn;
+        const system = halves.stable ?? turn.system;
+        return [
+            ...(system ? [{ role: "system", content: system }] : []),
+            ...(turn.transcript ?? []).map((message) => ({
+                role: message.role,
+                content: message.text,
+            })),
+            { role: "user", content: options.computerUse ? chatUserContent(userTurn) : userTurn.text },
+        ];
+    };
     const sendTurn = async (turn) => {
         if (!options.apiKey)
             throw new Error(options.missingKeyError);
         if (active.has(turn.threadId))
             throw new Error("a turn is already running on this thread");
+        if (options.computerUse && (turn.images?.length || turn.integrations?.computer || turn.integrations?.localComputer || turn.integrations?.browser))
+            assertImageTransport(options.apiUrl);
         const turnId = newId();
         const abort = new AbortController();
         const messages = messagesFor(turn);
+        const retainImages = chatImageBudget();
+        for (const message of messages)
+            retainImages(message.content);
         const model = turn.model || options.models().default;
-        const secrets = [options.apiKey];
+        const secrets = [options.apiKey, turn.integrations?.computer?.token, turn.integrations?.computer?.control?.token].filter((value) => Boolean(value));
         for (const integration of Object.values(turn.integrations ?? {})) {
             const entries = object(integration);
             const specs = entries && "command" in entries ? [entries] : Object.values(entries ?? {}).map(object);
@@ -270,10 +317,23 @@ export function createOpenAIChatRuntime(options) {
                 ...base(turn.threadId, turnId), type: "request.resolved", requestId: ask.id,
                 behavior: allowed ? "allow" : "deny", source,
             }),
+            openQuestion: (ask, questions) => {
+                const choices = questionChoices(questions);
+                emit({
+                    ...base(turn.threadId, turnId), type: "request.opened", requestType: "question",
+                    requestId: ask.id, tool: ask.tool, summary: ask.summary,
+                    questions, ...(choices ? { choices } : {}),
+                });
+            },
+            resolvedQuestion: (ask, answered, source) => emit({
+                ...base(turn.threadId, turnId), type: "request.resolved", requestId: ask.id,
+                behavior: answered ? "answer" : "deny", source,
+            }),
         });
         let resolveDone;
         const done = new Promise((resolve) => { resolveDone = resolve; });
-        active.set(turn.threadId, { abort, turnId, done, approval });
+        const turnEntry = { abort, turnId, done, approval, asides: [] };
+        active.set(turn.threadId, turnEntry);
         emit({ ...base(turn.threadId, turnId), type: "turn.started" });
         emit({ ...base(turn.threadId, turnId), type: "session.started", sessionId: null, model });
         void (async () => {
@@ -287,9 +347,25 @@ export function createOpenAIChatRuntime(options) {
             const denials = [];
             const seenCalls = new Set();
             try {
-                tools = await mountChatTools(options.tools === false ? undefined : turn.integrations, abort.signal);
+                tools = await mountChatTools(options.tools === false ? undefined : turn.integrations, abort.signal, options.computerUse);
+                let optionalQuestionOnly = options.tools !== false && tools.definitions.length === 0;
+                // The runtime's one built-in tool rides the same list: ask_user is
+                // how a chat-completions engine reaches a person. An MCP server that
+                // squats the name cannot shadow it — dispatch intercepts the name
+                // before validate — but the definition is then skipped so the list
+                // never advertises two.
+                if (options.tools !== false && !tools.definitions.some((definition) => definition.function.name === ASK_USER_TOOL)) {
+                    tools.definitions.push(ASK_USER_TOOL_DEFINITION);
+                }
                 for (let round = 0; round < 16; round++) {
                     abort.signal.throwIfAborted();
+                    // Drain parked mid-turn input here, before the next completion
+                    // request: the previous round's tool results are complete, so a
+                    // user message lands on a consistent array (never inside a tool
+                    // batch) and the provider sees it as the newest input.
+                    const parked = turnEntry.asides.splice(0);
+                    if (parked.length)
+                        messages.push({ role: "user", content: parked.join("\n\n") });
                     native("out", options.nativeLog.outgoing(turn, messages, model));
                     let attempt = 0;
                     let completion;
@@ -332,6 +408,15 @@ export function createOpenAIChatRuntime(options) {
                         catch (value) {
                             const error = asError(value);
                             const verdict = classifyError(error);
+                            // Only our optional question changed a previously plain request.
+                            // A structured parameter rejection has executed nothing; never
+                            // downgrade mounted tools, streamed output or a handled call.
+                            if (optionalQuestionOnly && round === 0 && !streamed && !seenCalls.size && !abort.signal.aborted &&
+                                error instanceof UnsupportedChatToolsError && verdict.reason === "invalid_request") {
+                                optionalQuestionOnly = false;
+                                tools.definitions.length = 0;
+                                continue;
+                            }
                             // Once a call has been handled, never replay it through a turn retry.
                             if (options.retryScale === undefined || abort.signal.aborted || streamed || seenCalls.size ||
                                 error instanceof ChatProtocolError || !verdict.transient || attempt >= RETRY_MAX_ATTEMPTS - 1)
@@ -381,6 +466,7 @@ export function createOpenAIChatRuntime(options) {
                         ...(completion.protocolReasoning ? { reasoning_content: completion.protocolReasoning } : {}),
                         ...(completion.protocolReasoningDetails.length ? { reasoning_details: completion.protocolReasoningDetails } : {}),
                     });
+                    const screenshotParts = [];
                     for (const call of completion.toolCalls) {
                         abort.signal.throwIfAborted();
                         let result;
@@ -396,26 +482,66 @@ export function createOpenAIChatRuntime(options) {
                             }
                             if (!object(args))
                                 throw new ChatProtocolError("tool arguments must be a JSON object");
-                            tools.validate(call.function.name, args);
                             const inputPreview = preview(args);
-                            // Full access is the person's explicit grant to answer every
-                            // prompt. This runtime has no provider reviewer to hand it to,
-                            // so it is honoured here: without it every single tool call on
-                            // an OpenAI-compatible engine stops for a card, and a Chief's
-                            // delegated Full access cannot help either.
-                            const allowed = turn.approvalMode === "full"
-                                || await approval.ask(call.function.name, inputPreview ?? "This tool has no arguments.");
-                            abort.signal.throwIfAborted();
-                            emit({ ...base(turn.threadId, turnId), type: "item.started", itemType: "tool", itemId: call.id,
-                                title: call.function.name, ...(inputPreview ? { input: inputPreview } : {}),
-                            });
-                            started = true;
-                            if (allowed) {
-                                result = await tools.execute(call.function.name, args, abort.signal);
+                            if (call.function.name === ASK_USER_TOOL) {
+                                // A question is the person's card, not a permission, so it is
+                                // handled before the gate below: under Full access that gate
+                                // would auto-run an unanswered ask, and under Ask it would
+                                // render Allow/Deny over a question nobody can answer that way.
+                                const questions = parseAskQuestions(args);
+                                abort.signal.throwIfAborted();
+                                emit({ ...base(turn.threadId, turnId), type: "item.started", itemType: "tool", itemId: call.id,
+                                    title: call.function.name, ...(inputPreview ? { input: inputPreview } : {}),
+                                });
+                                started = true;
+                                if (!questions) {
+                                    denials.push(ASK_USER_TOOL);
+                                    result = { ok: false, text: "The ask_user arguments are malformed: pass a JSON object with a questions array of one to six questions, each with a question string and at most twelve options carrying a label. Ask again with valid arguments." };
+                                }
+                                else {
+                                    // The tool result is the card's Q:/A: reply verbatim — never
+                                    // a summary, and never words the person did not send.
+                                    const answer = await approval.question(ASK_USER_TOOL, askQuestionSummary(questions), questions);
+                                    abort.signal.throwIfAborted();
+                                    if (answer === null) {
+                                        denials.push(ASK_USER_TOOL);
+                                        result = { ok: false, text: "The person did not answer this question. Do not guess an answer; ask again later or proceed without it." };
+                                    }
+                                    else {
+                                        result = { ok: true, text: answer };
+                                    }
+                                }
                             }
                             else {
-                                denials.push(call.function.name);
-                                result = { ok: false, text: "Permission denied or expired; the tool was not executed." };
+                                tools.validate(call.function.name, args);
+                                // Full access is the person's explicit grant to answer every
+                                // prompt. This runtime has no provider reviewer to hand it to,
+                                // so it is honoured here: without it every single tool call on
+                                // an OpenAI-compatible engine stops for a card, and a Chief's
+                                // delegated Full access cannot help either.
+                                const allowed = turn.approvalMode === "full"
+                                    || await approval.ask(call.function.name, inputPreview ?? "This tool has no arguments.");
+                                abort.signal.throwIfAborted();
+                                emit({ ...base(turn.threadId, turnId), type: "item.started", itemType: "tool", itemId: call.id,
+                                    title: call.function.name, ...(inputPreview ? { input: inputPreview } : {}),
+                                });
+                                started = true;
+                                if (allowed) {
+                                    result = await tools.execute(call.function.name, args, abort.signal);
+                                    if (result.images?.length) {
+                                        try {
+                                            assertImageTransport(options.apiUrl);
+                                            retainImages(result.images);
+                                        }
+                                        catch (error) {
+                                            throw new ChatToolSessionError(asError(error).message);
+                                        }
+                                    }
+                                }
+                                else {
+                                    denials.push(call.function.name);
+                                    result = { ok: false, text: "Permission denied or expired; the tool was not executed." };
+                                }
                             }
                         }
                         catch (error) {
@@ -433,10 +559,13 @@ export function createOpenAIChatRuntime(options) {
                         if (!result.ok)
                             toolFailed = true;
                         messages.push({ role: "tool", tool_call_id: call.id, content: JSON.stringify({ ok: result.ok, result: text }) });
+                        screenshotParts.push(...chatToolImages(call.id, result.images));
                         abort.signal.throwIfAborted();
                         if (fatal)
                             throw fatal;
                     }
+                    if (screenshotParts.length)
+                        messages.push({ role: "user", content: screenshotParts });
                 }
                 if (!ok)
                     throw new ChatProtocolError("model-call limit reached before a final response");
@@ -488,7 +617,12 @@ export function createOpenAIChatRuntime(options) {
             : { state: "unavailable", reason: options.unavailableReason },
         adapter: {
             provider: options.driverKind,
-            capabilities: { sessionModelSwitch: "in-session", customMcp: options.tools !== false, agentsMcp: options.tools !== false, composioMcp: options.tools !== false },
+            capabilities: { ...(options.computerUse ? { computerMcp: options.tools !== false, cloudComputerMcp: options.tools !== false, localComputerMcp: options.tools !== false,
+                    browserMcp: options.tools !== false, nativeImageInput: true, images: true } : {}),
+                sessionModelSwitch: "in-session", customMcp: options.tools !== false, agentsMcp: options.tools !== false, composioMcp: options.tools !== false,
+                // The runtime owns the whole tool loop, so it can always take a
+                // user message mid-turn: park it, deliver before the next completion.
+                queueing: true },
             sendTurn,
             interruptTurn: async (threadId, turnId) => {
                 const turn = active.get(threadId);
@@ -497,7 +631,20 @@ export function createOpenAIChatRuntime(options) {
                 turn.abort.abort();
                 await turn.done;
             },
-            respondToRequest: async (threadId, requestId, decision) => active.get(threadId)?.approval.answer(requestId, decision.behavior) ?? "unavailable",
+            respondToRequest: async (threadId, requestId, decision) => active.get(threadId)?.approval.answer(requestId, decision.behavior, decision.message) ?? "unavailable",
+            steer: async (threadId, text) => {
+                // This engine has no external session to respect — parking the
+                // words in the live turn's entry IS delivery into the loop, so a
+                // successful park is "steered" and a missing turn is the only
+                // refusal. The words ride the array at the next round boundary;
+                // if the turn settles first, the harness's transcript record
+                // keeps them for the next turn (they are never re-sent here).
+                const running = active.get(threadId);
+                if (!running)
+                    return "refused";
+                running.asides.push(text);
+                return "steered";
+            },
             hasSession: (threadId) => active.has(threadId),
             stopAll: async () => {
                 const turns = [...active.values()];

package/dist-server/server/drivers/openai-compat.js

@@ -133,6 +133,7 @@ export const OpenAICompatDriver = {
             apiKey,
             apiUrl: config.url,
             tools: config.tools,
+            computerUse: true,
             models: () => catalog,
             refreshModels: fetchModels,
             requestBody: (model, messages, stream) => ({

package/dist-server/server/drivers/permission-command.js

@@ -0,0 +1,35 @@
+import { realpathSync } from "node:fs";
+import { isAbsolute } from "node:path";
+/** Approval identity preserves every command byte, including whitespace and
+ * newlines. Callers must supply native shell input, never a display preview. */
+export function permissionCommand(command, cwd) {
+    if (typeof command !== "string" || !command.trim() || command.includes("\0"))
+        return undefined;
+    if (typeof cwd !== "string" || !isAbsolute(cwd) || cwd.includes("\0"))
+        return undefined;
+    return { command, cwd };
+}
+export function permissionLaunchCwd(cwd) {
+    try {
+        return realpathSync(cwd);
+    }
+    catch {
+        return undefined;
+    }
+}
+/** ACP's rawInput is provider-specific. Accept a complete command string and
+ * unambiguous absolute directory fields. A relative/invalid/conflicting
+ * directory is not evidence of where the agent will execute it. */
+export function acpPermissionCommand(rawInput, launchCwd) {
+    if (!rawInput || typeof rawInput !== "object" || Array.isArray(rawInput))
+        return undefined;
+    const input = rawInput;
+    const directories = ["cwd", "workdir", "working_directory", "directory", "dir"]
+        .filter((key) => input[key] !== undefined)
+        .map((key) => input[key]);
+    if (directories.some((value) => typeof value !== "string" || !isAbsolute(value) || value.includes("\0")))
+        return undefined;
+    if (new Set(directories).size > 1)
+        return undefined;
+    return permissionCommand(input.command, directories[0] ?? launchCwd);
+}

package/dist-server/server/drivers/phone-proxy.js

@@ -6,6 +6,10 @@ import { createInterface } from "node:readline";
 import { delimiter, join } from "node:path";
 import { homedir } from "node:os";
 const MAX_ADB_OUTPUT = 16 * 1024 * 1024;
+// Mirrors the shared-computer blockedReason shape: name the holder's
+// situation, say the call did not happen, and tell the model what beats a
+// blind retry once the phone frees up.
+const PHONE_BUSY_TEXT = "Another thread is using the phone. This call was not performed. Pause phone work until that thread finishes, then read the screen again before acting.";
 const PACKAGE = /^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)+$/;
 const COMPONENT = /^([A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)+)\/\S+$/;
 const SAFE_TEXT = /^[A-Za-z0-9 _.,@-]+$/;
@@ -240,6 +244,43 @@ const TOOLS = [
     { name: "press", description: "Press an Android navigation or keyboard key.", inputSchema: { type: "object", properties: { key: { type: "string", enum: Object.keys(KEYCODES) }, serial: { type: "string" } }, required: ["key"] } },
 ];
 const textResult = (text, isError = false) => ({ content: [{ type: "text", text }], ...(isError ? { isError: true } : {}) });
+// The harness reserves the physical phone for one turn at a time, but only
+// from the first real tool call (issue #1663): trigger-term matching decides
+// whether these tools are mounted, never who holds the device. Revalidate
+// every call: retained MCP processes can outlive their turn, whose claim and
+// capability are released at settlement. Reclaiming for the same live owner
+// is idempotent; a blocked caller can retry once the holding turn settles.
+export function createPhoneClaim(env = process.env, fetchImpl = fetch) {
+    const url = env.OMB_HARNESS_URL?.trim();
+    const token = env.OMB_PHONE_TOKEN?.trim();
+    if (!url || !token) {
+        // Spawned outside the harness (development, direct debugging): no
+        // exclusivity to enforce, exactly the behavior this proxy always had.
+        return async () => ({ ok: true });
+    }
+    return async () => {
+        try {
+            const response = await fetchImpl(new URL("/api/internal/phone/claim", url), {
+                method: "POST",
+                redirect: "error",
+                headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
+                body: "{}",
+                signal: AbortSignal.timeout(10_000),
+            });
+            if (response.ok)
+                return { ok: true };
+            if (response.status === 409)
+                return { ok: false, message: PHONE_BUSY_TEXT };
+            if (response.status === 401 || response.status === 403) {
+                return { ok: false, message: "This turn no longer has phone access. Start a new turn to use the phone." };
+            }
+            return { ok: false, message: "OpenMausBot could not reserve the phone for this call. This call was not performed; try again, or start a new turn if it keeps failing." };
+        }
+        catch {
+            return { ok: false, message: "OpenMausBot could not be reached to reserve the phone. This call was not performed; try again." };
+        }
+    };
+}
 async function callTool(name, args) {
     if (name === "status") {
         const adb = resolveAdbPath();
@@ -334,7 +375,7 @@ async function callTool(name, args) {
 const send = (message) => process.stdout.write(`${JSON.stringify(message)}\n`);
 const ok = (id, result) => send({ jsonrpc: "2.0", id, result });
 const rpcError = (id, code, message) => send({ jsonrpc: "2.0", id, error: { code, message } });
-async function handle(message) {
+async function handle(message, ensureClaim) {
     const id = message.id;
     const method = message.method;
     const params = (message.params ?? {});
@@ -350,6 +391,9 @@ async function handle(message) {
         const name = String(params.name ?? "");
         if (!TOOLS.some((tool) => tool.name === name))
             return rpcError(id, -32602, `Unknown tool: ${name}`);
+        const claim = await ensureClaim();
+        if (!claim.ok)
+            return ok(id, textResult(claim.message, true));
         try {
             return ok(id, await callTool(name, (params.arguments ?? {})));
         }
@@ -361,11 +405,12 @@ async function handle(message) {
         rpcError(id, -32601, `Method not found: ${String(method)}`);
 }
 if (process.argv[1] && existsSync(process.argv[1]) && /phone-proxy\.(?:ts|js)$/.test(process.argv[1])) {
+    const ensureClaim = createPhoneClaim();
     const lines = createInterface({ input: process.stdin, terminal: false });
     lines.on("line", (line) => {
         try {
             const message = JSON.parse(line);
-            void handle(message).catch((error) => rpcError(message.id, -32603, error instanceof Error ? error.message : String(error)));
+            void handle(message, ensureClaim).catch((error) => rpcError(message.id, -32603, error instanceof Error ? error.message : String(error)));
         }
         catch { }
     });