codex-computer-use-mcp 0.2.0 → 0.3.1

package/integrations/pi/index.ts

@@ -4,52 +4,95 @@ import { getAgentDir, truncateHead, type ExtensionAPI } from "@earendil-works/pi
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type TSchema } from "typebox";
 import { executeDirectTool, getDirectStatus } from "../../dist/direct-service.js";
-import type { DirectMethod } from "../../dist/tools.js";
+import type {
+  DirectBrokerElicitationRequest,
+  DirectBrokerElicitationResponse,
+} from "../../dist/direct-broker.js";
+import { OFFICIAL_TOOL_METADATA, type DirectMethod } from "../../dist/tools.js";
 
-const App = Type.String({ minLength: 1, maxLength: 500, description: "App name, full app path, or unambiguous bundle identifier" });
-const Element = Type.String({ minLength: 1, maxLength: 200, description: "Element identifier from computer_use_get_app_state" });
-const Coordinate = Type.Number({ minimum: 0, maximum: 1_000_000 });
+const App = Type.String({ description: "App name, full app path, or unambiguous bundle identifier" });
 
 const ToolParameters: Record<DirectMethod, TSchema> = {
   list_apps: Type.Object({}, { additionalProperties: false }),
   get_app_state: Type.Object({ app: App }, { additionalProperties: false }),
   click: Type.Object({
     app: App,
-    click_count: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })),
-    element_index: Type.Optional(Element),
-    mouse_button: Type.Optional(StringEnum(["left", "right", "middle"] as const)),
-    x: Type.Optional(Coordinate),
-    y: Type.Optional(Coordinate),
+    click_count: Type.Optional(Type.Integer({ description: "Number of clicks. Defaults to 1" })),
+    element_index: Type.Optional(Type.String({ description: "Element index to click" })),
+    mouse_button: Type.Optional(StringEnum(["left", "right", "middle"] as const, { description: "Mouse button to click. Defaults to left." })),
+    x: Type.Optional(Type.Number({ description: "X coordinate in screenshot pixel coordinates" })),
+    y: Type.Optional(Type.Number({ description: "Y coordinate in screenshot pixel coordinates" })),
   }, { additionalProperties: false }),
-  perform_secondary_action: Type.Object({ app: App, element_index: Element, action: Type.String({ minLength: 1, maxLength: 200 }) }, { additionalProperties: false }),
-  set_value: Type.Object({ app: App, element_index: Element, value: Type.String({ maxLength: 20_000 }) }, { additionalProperties: false }),
-  select_text: Type.Object({
+  perform_secondary_action: Type.Object({
+    app: App,
+    element_index: Type.String({ description: "Element identifier" }),
+    action: Type.String({ description: "Secondary accessibility action name" }),
+  }, { additionalProperties: false }),
+  set_value: Type.Object({
     app: App,
-    element_index: Element,
-    text: Type.String({ minLength: 1, maxLength: 20_000 }),
-    prefix: Type.Optional(Type.String({ maxLength: 2_000 })),
-    selection: Type.Optional(StringEnum(["text", "cursor_before", "cursor_after"] as const)),
-    suffix: Type.Optional(Type.String({ maxLength: 2_000 })),
+    element_index: Type.String({ description: "Element identifier" }),
+    value: Type.String({ description: "Value to assign" }),
+  }, { additionalProperties: false }),
+  select_text: Type.Object({
+    app: Type.String({ description: "App name or bundle identifier" }),
+    element_index: Type.String({ description: "Text element identifier" }),
+    text: Type.String({ description: "Target text as shown in the accessibility tree" }),
+    prefix: Type.Optional(Type.String({ description: "Optional text immediately before the target, used to disambiguate repeated matches" })),
+    selection: Type.Optional(StringEnum(["text", "cursor_before", "cursor_after"] as const, { description: "Whether to select the text or place the cursor before or after it. Defaults to text." })),
+    suffix: Type.Optional(Type.String({ description: "Optional text immediately after the target, used to disambiguate repeated matches" })),
   }, { additionalProperties: false }),
   scroll: Type.Object({
     app: App,
-    element_index: Element,
-    direction: StringEnum(["up", "down", "left", "right"] as const),
-    pages: Type.Optional(Type.Number({ exclusiveMinimum: 0, maximum: 100 })),
+    element_index: Type.String({ description: "Element identifier" }),
+    direction: Type.String({ description: "Scroll direction: up, down, left, or right" }),
+    pages: Type.Optional(Type.Number({ description: "Number of pages to scroll. Fractional values are supported. Defaults to 1" })),
   }, { additionalProperties: false }),
-  drag: Type.Object({ app: App, from_x: Coordinate, from_y: Coordinate, to_x: Coordinate, to_y: Coordinate }, { additionalProperties: false }),
-  press_key: Type.Object({ app: App, key: Type.String({ minLength: 1, maxLength: 100, description: "Key or combination; common aliases such as CMD+A are normalized to the official xdotool-style Meta_L+a form" }) }, { additionalProperties: false }),
-  type_text: Type.Object({ app: App, text: Type.String({ maxLength: 20_000 }) }, { additionalProperties: false }),
+  drag: Type.Object({
+    app: App,
+    from_x: Type.Number({ description: "Start X coordinate" }),
+    from_y: Type.Number({ description: "Start Y coordinate" }),
+    to_x: Type.Number({ description: "End X coordinate" }),
+    to_y: Type.Number({ description: "End Y coordinate" }),
+  }, { additionalProperties: false }),
+  press_key: Type.Object({ app: App, key: Type.String({ description: "Key or key combination to press" }) }, { additionalProperties: false }),
+  type_text: Type.Object({ app: App, text: Type.String({ description: "Literal text to type" }) }, { additionalProperties: false }),
 };
 
 function toolDescription(method: DirectMethod): string {
-  return `Call the official signed Computer Use ${method} capability directly through the unrestricted no-permissions interface. Pi supplies the typed arguments itself; there is no wrapper approval prompt, mode gate, nested model, planner, prompt, or separate model-token usage.`;
+  return OFFICIAL_TOOL_METADATA[method].description;
 }
 
 function titleFor(method: DirectMethod): string {
   return method.split("_").map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`).join(" ");
 }
 
+const INSPECTION_METHODS = new Set<DirectMethod>(["list_apps", "get_app_state"]);
+export const INSPECTION_TOOL_NAMES = [
+  "computer_use_list_apps",
+  "computer_use_get_app_state",
+] as const;
+export const INTERACTION_TOOL_NAMES = [
+  "computer_use_click",
+  "computer_use_perform_secondary_action",
+  "computer_use_set_value",
+  "computer_use_select_text",
+  "computer_use_scroll",
+  "computer_use_drag",
+  "computer_use_press_key",
+  "computer_use_type_text",
+] as const;
+
+export function setInitialComputerUseTools(pi: Pick<ExtensionAPI, "getActiveTools" | "setActiveTools">): void {
+  const interactionTools = new Set<string>(INTERACTION_TOOL_NAMES);
+  const preserved = pi.getActiveTools().filter((name) => !interactionTools.has(name));
+  pi.setActiveTools([...new Set([...preserved, ...INSPECTION_TOOL_NAMES])]);
+}
+
+export function activateInteractionTools(pi: Pick<ExtensionAPI, "getActiveTools" | "setActiveTools">): void {
+  const active = pi.getActiveTools();
+  pi.setActiveTools([...new Set([...active, ...INTERACTION_TOOL_NAMES])]);
+}
+
 function toPiContent(content: Array<Record<string, unknown>>): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
   const result: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = [];
   for (const block of content) {
@@ -75,6 +118,74 @@ function errorText(content: Array<Record<string, unknown>>): string {
     .slice(0, 2_000) || "Official Computer Use returned an error";
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function initialFormContent(schema: unknown): Record<string, unknown> {
+  if (!isRecord(schema) || !isRecord(schema.properties)) return {};
+  const content: Record<string, unknown> = {};
+  for (const [key, rawProperty] of Object.entries(schema.properties)) {
+    if (!isRecord(rawProperty)) continue;
+    if (rawProperty.default !== undefined) content[key] = rawProperty.default;
+    else if (Array.isArray(rawProperty.enum) && rawProperty.enum.length > 0) content[key] = rawProperty.enum[0];
+    else if (rawProperty.type === "boolean") content[key] = false;
+    else if (rawProperty.type === "number" || rawProperty.type === "integer") content[key] = 0;
+    else if (rawProperty.type === "array") content[key] = [];
+    else content[key] = "";
+  }
+  return content;
+}
+
+export async function handleOfficialElicitation(
+  request: DirectBrokerElicitationRequest,
+  ctx: { hasUI: boolean; ui: {
+    select(title: string, options: string[]): Promise<string | undefined>;
+    editor(title: string, prefill?: string): Promise<string | undefined>;
+    notify(message: string, type?: "info" | "warning" | "error"): void;
+  } },
+  openUrl: (url: string) => Promise<boolean>,
+): Promise<DirectBrokerElicitationResponse> {
+  if (!ctx.hasUI) return { action: "cancel" };
+  const message = typeof request.message === "string" ? request.message : "Official Computer Use requests input";
+
+  if (request.mode === "url") {
+    if (typeof request.url !== "string") return { action: "cancel" };
+    const choice = await ctx.ui.select(`${message}\n${request.url}`, ["Open URL", "Decline", "Cancel"]);
+    if (choice === "Decline") return { action: "decline" };
+    if (choice !== "Open URL") return { action: "cancel" };
+    if (!await openUrl(request.url)) {
+      ctx.ui.notify("Could not open the official Computer Use URL.", "error");
+      return { action: "cancel" };
+    }
+    return { action: "accept" };
+  }
+
+  if (request.mode !== undefined && request.mode !== "form" && request.mode !== "openai/form") {
+    ctx.ui.notify(`Official Computer Use sent an unsupported elicitation mode: ${request.mode}`, "warning");
+    return { action: "cancel" };
+  }
+  const openAiForm = request.mode === "openai/form";
+  if (request.requestedSchema === undefined || (!openAiForm && !isRecord(request.requestedSchema))) return { action: "cancel" };
+  const choice = await ctx.ui.select(message, ["Respond", "Decline", "Cancel"]);
+  if (choice === "Decline") return { action: "decline" };
+  if (choice !== "Respond") return { action: "cancel" };
+  let prefill = JSON.stringify(initialFormContent(request.requestedSchema), null, 2);
+  const title = `${message}\nSchema: ${JSON.stringify(request.requestedSchema)}`;
+  while (true) {
+    const edited = await ctx.ui.editor(title, prefill);
+    if (edited === undefined) return { action: "cancel" };
+    prefill = edited;
+    try {
+      const content = JSON.parse(edited);
+      if (!openAiForm && !isRecord(content)) throw new Error("Response must be a JSON object");
+      return { action: "accept", content };
+    } catch (error) {
+      ctx.ui.notify(error instanceof Error ? error.message : "Response must be valid JSON", "error");
+    }
+  }
+}
+
 export default function directComputerUse(pi: ExtensionAPI) {
   const stateRoot = process.env.CODEX_COMPUTER_USE_HOME || path.join(getAgentDir(), "direct-computer-use");
 
@@ -89,27 +200,39 @@ export default function directComputerUse(pi: ExtensionAPI) {
   const methods = Object.keys(ToolParameters) as DirectMethod[];
   for (const method of methods) {
     const piName = `computer_use_${method}`;
-    pi.registerTool({
-      name: piName,
-      label: titleFor(method),
-      description: toolDescription(method),
+    const inspectionPromptMetadata = INSPECTION_METHODS.has(method) ? {
       promptSnippet: `${titleFor(method)} through the official signed macOS Computer Use service`,
       promptGuidelines: [
         `${piName} is a direct typed tool: choose its arguments yourself; it does not invoke a nested planner or model.`,
-        `${piName} must use current element identifiers from computer_use_get_app_state; inspect again after UI state changes.`,
-        `${piName} must not be used for credentials, authentication, payments, external messages, or destructive actions without the user's explicit request; this wrapper will not open a permission prompt.`,
+        `Call computer_use_get_app_state once per assistant turn before interacting with an app.`,
       ],
+    } : {};
+    pi.registerTool({
+      name: piName,
+      label: titleFor(method),
+      description: toolDescription(method),
+      ...inspectionPromptMetadata,
       parameters: ToolParameters[method] as any,
-      async execute(_toolCallId, params, signal, onUpdate) {
+      async execute(_toolCallId, params, signal, onUpdate, ctx) {
         const response = await executeDirectTool(
           { method, arguments: params as Record<string, unknown> },
           {
             stateRoot,
             signal,
+            supportsOpenAiFormElicitation: true,
+            onElicitation: (request) => handleOfficialElicitation(
+              request,
+              ctx,
+              async (url) => {
+                const result = await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 });
+                return result.code === 0;
+              },
+            ),
             onProgress: (message) => onUpdate?.({ content: [{ type: "text", text: message }], details: { status: "running" } }),
           },
         );
         if (response.isError) throw new Error(errorText(response.content));
+        if (method === "get_app_state") activateInteractionTools(pi);
         return { content: toPiContent(response.content), details: response.details };
       },
       renderCall(args, theme) {
@@ -119,4 +242,6 @@ export default function directComputerUse(pi: ExtensionAPI) {
       },
     });
   }
+
+  pi.on("session_start", () => setInitialComputerUseTools(pi));
 }

package/integrations/pi/mcp.json.example

@@ -4,7 +4,7 @@
       "command": "npx",
       "args": [
         "-y",
-        "codex-computer-use-mcp@0.2.0"
+        "codex-computer-use-mcp@0.3.1"
       ],
       "lifecycle": "lazy",
       "requestTimeoutMs": 180000,

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "codex-computer-use-mcp",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "codex-computer-use-mcp",
-      "version": "0.2.0",
+      "version": "0.3.1",
       "license": "MIT",
       "os": [
         "darwin"
@@ -21,9 +21,9 @@
         "codex-computer-use-mcp": "dist/mcp-server.js"
       },
       "devDependencies": {
-        "@earendil-works/pi-ai": "0.80.6",
-        "@earendil-works/pi-coding-agent": "0.80.6",
-        "@earendil-works/pi-tui": "0.80.6",
+        "@earendil-works/pi-ai": "0.80.7",
+        "@earendil-works/pi-coding-agent": "0.80.7",
+        "@earendil-works/pi-tui": "0.80.7",
         "typebox": "1.1.38"
       },
       "engines": {
@@ -541,9 +541,9 @@
       }
     },
     "node_modules/@earendil-works/pi-ai": {
-      "version": "0.80.6",
-      "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.6.tgz",
-      "integrity": "sha512-7xfLk8sANBp+bpPEbjoOZTbPxsa+++b1JXAoSJsNa3vbs9AHHEclmvg54XLQcxH+fuwaeti/g2jeIfJ+mVYLpA==",
+      "version": "0.80.7",
+      "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.7.tgz",
+      "integrity": "sha512-8RLKLwe5TFM9kKFMNu/lTzveduq4GxZbnlG6ba8FAhLeb5wJP4zbj1eBumKBRvggpFQnW5R/Vo2a8zTlHsV9SQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -567,16 +567,16 @@
       }
     },
     "node_modules/@earendil-works/pi-coding-agent": {
-      "version": "0.80.6",
-      "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.80.6.tgz",
-      "integrity": "sha512-vcfD6tOk402isLl3Cm/qbn2O10TvgroMp1+/fEGM24ZdvETFCdOYv5VZ7m59EI5fPsjfSJh+CpQ5bhBrhfOg7g==",
+      "version": "0.80.7",
+      "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.80.7.tgz",
+      "integrity": "sha512-mxq3IClhdgmCrYiKuzKehs4QKCVJgKmlA70nUEzsgeNsGdxriT7o5sKQTCcxzVJkzDRMcHD/8tAP4/eGrV4gKQ==",
       "dev": true,
       "hasShrinkwrap": true,
       "license": "MIT",
       "dependencies": {
-        "@earendil-works/pi-agent-core": "^0.80.6",
-        "@earendil-works/pi-ai": "^0.80.6",
-        "@earendil-works/pi-tui": "^0.80.6",
+        "@earendil-works/pi-agent-core": "^0.80.7",
+        "@earendil-works/pi-ai": "^0.80.7",
+        "@earendil-works/pi-tui": "^0.80.7",
         "@silvia-odwyer/photon-node": "0.3.4",
         "chalk": "5.6.2",
         "cross-spawn": "7.0.6",
@@ -1066,13 +1066,12 @@
       }
     },
     "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-agent-core": {
-      "version": "0.80.6",
-      "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.6.tgz",
-      "integrity": "sha512-Lvn89ko42h5ETUb6Z0Ku6ldskEqXaTdQBYvSa0+7bdG9V6rUEpXptv5e0OVZ1HDcvi8s6/2lGCQWsxKX+DFHNw==",
+      "version": "0.80.7",
+      "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.7.tgz",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@earendil-works/pi-ai": "^0.80.6",
+        "@earendil-works/pi-ai": "^0.80.7",
         "ignore": "7.0.5",
         "typebox": "1.1.38",
         "yaml": "2.9.0"
@@ -1082,9 +1081,8 @@
       }
     },
     "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-ai": {
-      "version": "0.80.6",
-      "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.6.tgz",
-      "integrity": "sha512-7xfLk8sANBp+bpPEbjoOZTbPxsa+++b1JXAoSJsNa3vbs9AHHEclmvg54XLQcxH+fuwaeti/g2jeIfJ+mVYLpA==",
+      "version": "0.80.7",
+      "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.7.tgz",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -1108,9 +1106,8 @@
       }
     },
     "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-tui": {
-      "version": "0.80.6",
-      "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.6.tgz",
-      "integrity": "sha512-bSuzS4EVSqEPj/Qr/p9eqCESfKsGuDNbl77EGci8Iaqqt/C/XCBZL1MjXaxSWW1NsT5afjp/Cb0NTPzOLv/aPA==",
+      "version": "0.80.7",
+      "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.7.tgz",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -2543,9 +2540,9 @@
       }
     },
     "node_modules/@earendil-works/pi-tui": {
-      "version": "0.80.6",
-      "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.6.tgz",
-      "integrity": "sha512-bSuzS4EVSqEPj/Qr/p9eqCESfKsGuDNbl77EGci8Iaqqt/C/XCBZL1MjXaxSWW1NsT5afjp/Cb0NTPzOLv/aPA==",
+      "version": "0.80.7",
+      "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.7.tgz",
+      "integrity": "sha512-1B2++fLZfgI3XMzW2BTpuDuam2uyHnUUEmsOvi5R0Ne9RAt59WjFV0G8ozX6l1Xafa9P5Y3eT4aDtRr/v/CUTA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -3667,9 +3664,9 @@
       "license": "MIT"
     },
     "node_modules/fast-uri": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz",
-      "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz",
+      "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==",
       "funding": [
         {
           "type": "github",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-computer-use-mcp",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "Expose the official macOS Computer Use tools directly to Pi and MCP clients without a nested model.",
   "author": "Thomas Mustier",
   "license": "MIT",
@@ -35,6 +35,7 @@
     "ARCHITECTURE.md",
     "MIGRATION.md",
     "SECURITY.md",
+    "docs/adr",
     "CONTRIBUTING.md",
     "LICENSE"
   ],
@@ -73,9 +74,9 @@
     }
   },
   "devDependencies": {
-    "@earendil-works/pi-ai": "0.80.6",
-    "@earendil-works/pi-coding-agent": "0.80.6",
-    "@earendil-works/pi-tui": "0.80.6",
+    "@earendil-works/pi-ai": "0.80.7",
+    "@earendil-works/pi-coding-agent": "0.80.7",
+    "@earendil-works/pi-tui": "0.80.7",
     "typebox": "1.1.38"
   },
   "engines": {