codex-computer-use-mcp 0.3.4 → 0.4.0

package/integrations/pi/index.ts

@@ -1,237 +1,93 @@
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import path from "node:path";
-import { StringEnum } from "@earendil-works/pi-ai";
-import { getAgentDir, truncateHead, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import {
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+  formatSize,
+  getAgentDir,
+  truncateHead,
+  type ExtensionAPI,
+} from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
-import { Type, type TSchema } from "typebox";
-import { executeDirectTool, getDirectStatus } from "../../dist/direct-service.js";
+import { z } from "zod";
+import { getDirectStatus } from "../../dist/direct-service.js";
 import {
-  createOfficialDirectToolSession,
   type DirectBrokerElicitationRequest,
   type DirectBrokerElicitationResponse,
-  type OfficialDirectToolSession,
 } from "../../dist/direct-broker.js";
-import { OFFICIAL_TOOL_METADATA, type DirectMethod } from "../../dist/tools.js";
-
-const App = Type.String({ description: "App name, full app path, or unambiguous bundle identifier" });
-
-const ToolParameters: Record<DirectMethod, TSchema> = {
-  list_apps: Type.Object({}, { additionalProperties: false }),
-  get_app_state: Type.Object({ app: App }, { additionalProperties: false }),
-  click: Type.Object({
-    app: App,
-    click_count: Type.Optional(Type.Integer({ description: "Number of clicks. Defaults to 1" })),
-    element_index: Type.Optional(Type.String({ description: "Element index to click" })),
-    mouse_button: Type.Optional(StringEnum(["left", "right", "middle"] as const, { description: "Mouse button to click. Defaults to left." })),
-    x: Type.Optional(Type.Number({ description: "X coordinate in screenshot pixel coordinates" })),
-    y: Type.Optional(Type.Number({ description: "Y coordinate in screenshot pixel coordinates" })),
-  }, { additionalProperties: false }),
-  perform_secondary_action: Type.Object({
-    app: App,
-    element_index: Type.String({ description: "Element identifier" }),
-    action: Type.String({ description: "Secondary accessibility action name" }),
-  }, { additionalProperties: false }),
-  set_value: Type.Object({
-    app: App,
-    element_index: Type.String({ description: "Element identifier" }),
-    value: Type.String({ description: "Value to assign" }),
-  }, { additionalProperties: false }),
-  select_text: Type.Object({
-    app: Type.String({ description: "App name or bundle identifier" }),
-    element_index: Type.String({ description: "Text element identifier" }),
-    text: Type.String({ description: "Target text as shown in the accessibility tree" }),
-    prefix: Type.Optional(Type.String({ description: "Optional text immediately before the target, used to disambiguate repeated matches" })),
-    selection: Type.Optional(StringEnum(["text", "cursor_before", "cursor_after"] as const, { description: "Whether to select the text or place the cursor before or after it. Defaults to text." })),
-    suffix: Type.Optional(Type.String({ description: "Optional text immediately after the target, used to disambiguate repeated matches" })),
-  }, { additionalProperties: false }),
-  scroll: Type.Object({
-    app: App,
-    element_index: Type.String({ description: "Element identifier" }),
-    direction: Type.String({ description: "Scroll direction: up, down, left, or right" }),
-    pages: Type.Optional(Type.Number({ description: "Number of pages to scroll. Fractional values are supported. Defaults to 1" })),
-  }, { additionalProperties: false }),
-  drag: Type.Object({
-    app: App,
-    from_x: Type.Number({ description: "Start X coordinate" }),
-    from_y: Type.Number({ description: "Start Y coordinate" }),
-    to_x: Type.Number({ description: "End X coordinate" }),
-    to_y: Type.Number({ description: "End Y coordinate" }),
-  }, { additionalProperties: false }),
-  press_key: Type.Object({ app: App, key: Type.String({ description: "Key or key combination to press" }) }, { additionalProperties: false }),
-  type_text: Type.Object({ app: App, text: Type.String({ description: "Literal text to type" }) }, { additionalProperties: false }),
-};
+import { DirectSessionExecutor } from "../../dist/session-executor.js";
+import {
+  TOOL_INPUT_SCHEMAS,
+  TOOL_METADATA,
+  COMPUTER_USE_METHODS,
+  type DirectMethod,
+  type JsonObject,
+} from "../../dist/tools.js";
 
-function toolDescription(method: DirectMethod): string {
-  return OFFICIAL_TOOL_METADATA[method].description;
-}
+const jsonObjectSchema = z.record(z.string(), z.json());
+const toolNames = COMPUTER_USE_METHODS.map((method) => `computer_use_${method}`);
 
 function titleFor(method: DirectMethod): string {
   return method.split("_").map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`).join(" ");
 }
 
-const INSPECTION_METHODS = new Set<DirectMethod>(["list_apps", "get_app_state"]);
-export const INSPECTION_TOOL_NAMES = [
-  "computer_use_list_apps",
-  "computer_use_get_app_state",
-] as const;
-export const INTERACTION_TOOL_NAMES = [
-  "computer_use_click",
-  "computer_use_perform_secondary_action",
-  "computer_use_set_value",
-  "computer_use_select_text",
-  "computer_use_scroll",
-  "computer_use_drag",
-  "computer_use_press_key",
-  "computer_use_type_text",
-] as const;
-
-export const COMPUTER_USE_TOOL_NAMES = [
-  ...INSPECTION_TOOL_NAMES,
-  ...INTERACTION_TOOL_NAMES,
-] as const;
-
-export function setInitialComputerUseTools(pi: Pick<ExtensionAPI, "getActiveTools" | "setActiveTools">): void {
-  pi.setActiveTools([...new Set([...pi.getActiveTools(), ...COMPUTER_USE_TOOL_NAMES])]);
-}
-
-interface SessionExecutorDependencies {
-  createSession?: typeof createOfficialDirectToolSession;
-  executeTool?: typeof executeDirectTool;
+interface PiContentResult {
+  content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }>;
+  fullOutputPath?: string;
 }
 
-/** Keep the signed app-server/client alive across the required state -> action pair. */
-export class PiDirectSessionExecutor {
-  private session: OfficialDirectToolSession | undefined;
-  private queue: Promise<void> = Promise.resolve();
-  private readonly dependencies: SessionExecutorDependencies;
-
-  constructor(dependencies: SessionExecutorDependencies = {}) {
-    this.dependencies = dependencies;
-  }
-
-  private async closeSession(): Promise<void> {
-    const session = this.session;
-    this.session = undefined;
-    await session?.close();
-  }
-
-  async execute(
-    method: DirectMethod,
-    params: Record<string, unknown>,
-    dependencies: Parameters<typeof executeDirectTool>[1],
-  ): Promise<Awaited<ReturnType<typeof executeDirectTool>>> {
-    const previous = this.queue;
-    let release!: () => void;
-    this.queue = new Promise<void>((resolve) => { release = resolve; });
-    await previous;
-    try {
-      const executeTool = this.dependencies.executeTool ?? executeDirectTool;
-      if (method === "list_apps") return await executeTool({ method, arguments: params }, dependencies);
-      if (method === "get_app_state") {
-        await this.closeSession();
-        const createSession = this.dependencies.createSession ?? createOfficialDirectToolSession;
-        try {
-          const response = await executeTool(
-            { method, arguments: params },
-            {
-              ...dependencies,
-              callTool: async (directMethod, args, options) => {
-                this.session = await createSession({ supportsOpenAiFormElicitation: true });
-                return this.session.call(directMethod, args, options);
-              },
-            },
-          );
-          if (response.isError) await this.closeSession();
-          return response;
-        } catch (error) {
-          await this.closeSession();
-          throw error;
-        }
-      }
-      const session = this.session;
-      try {
-        return await executeTool(
-          { method, arguments: params },
-          session ? {
-            ...dependencies,
-            callTool: async (directMethod, args, options) => {
-              const result = await session.call(directMethod, args, options);
-              await this.closeSession();
-              return { ...result, brokerCleanupVerified: true };
-            },
-          } : dependencies,
-        );
-      } finally {
-        await this.closeSession();
-      }
-    } finally {
-      release();
-    }
-  }
-
-  async close(): Promise<void> {
-    await this.queue;
-    await this.closeSession();
+export async function toPiContent(content: JsonObject[]): Promise<PiContentResult> {
+  const textBlocks = content
+    .filter((block) => block.type !== "image")
+    .map((block) => String(block.text ?? ""));
+  const truncations = textBlocks.map((text) => truncateHead(text, {
+    maxLines: DEFAULT_MAX_LINES,
+    maxBytes: DEFAULT_MAX_BYTES,
+  }));
+  let fullOutputPath: string | undefined;
+  if (truncations.some((result) => result.truncated)) {
+    const tempDir = await mkdtemp(path.join(tmpdir(), "pi-computer-use-"));
+    fullOutputPath = path.join(tempDir, "output.txt");
+    await writeFile(fullOutputPath, textBlocks.join("\n\n"), { encoding: "utf8", mode: 0o600 });
   }
-}
 
-function toPiContent(content: Array<Record<string, unknown>>): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
-  const result: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = [];
+  const result: PiContentResult = { content: [] };
+  let textIndex = 0;
   for (const block of content) {
     if (block.type === "image") {
-      result.push({ type: "image", data: String(block.data), mimeType: String(block.mimeType) });
+      result.content.push({ type: "image", data: String(block.data), mimeType: String(block.mimeType) });
       continue;
     }
-    const original = String(block.text ?? "");
-    const truncated = truncateHead(original, { maxLines: 2_000, maxBytes: 50 * 1024 });
+    const truncated = truncations[textIndex++];
     const suffix = truncated.truncated
-      ? `\n\n[Official Computer Use text truncated in-memory: ${truncated.outputLines}/${truncated.totalLines} lines, ${truncated.outputBytes}/${truncated.totalBytes} bytes. No full-output file was written.]`
+      ? `\n\n[Official Computer Use text truncated: showing ${truncated.outputLines} of ${truncated.totalLines} lines (${formatSize(truncated.outputBytes)} of ${formatSize(truncated.totalBytes)}). Full output saved to: ${fullOutputPath}]`
       : "";
-    result.push({ type: "text", text: `${truncated.content}${suffix}` });
+    result.content.push({ type: "text", text: `${truncated.content}${suffix}` });
   }
+  if (fullOutputPath) result.fullOutputPath = fullOutputPath;
   return result;
 }
 
-function errorText(content: Array<Record<string, unknown>>): string {
-  return content
-    .filter((block) => block.type === "text")
-    .map((block) => String(block.text ?? ""))
-    .join("\n")
-    .slice(0, 2_000) || "Official Computer Use returned an error";
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
-}
-
-function initialFormContent(schema: unknown): Record<string, unknown> {
-  if (!isRecord(schema) || !isRecord(schema.properties)) return {};
-  const content: Record<string, unknown> = {};
-  for (const [key, rawProperty] of Object.entries(schema.properties)) {
-    if (!isRecord(rawProperty)) continue;
-    if (rawProperty.default !== undefined) content[key] = rawProperty.default;
-    else if (Array.isArray(rawProperty.enum) && rawProperty.enum.length > 0) content[key] = rawProperty.enum[0];
-    else if (rawProperty.type === "boolean") content[key] = false;
-    else if (rawProperty.type === "number" || rawProperty.type === "integer") content[key] = 0;
-    else if (rawProperty.type === "array") content[key] = [];
-    else content[key] = "";
-  }
-  return content;
+interface PiElicitationContext {
+  hasUI: boolean;
+  ui: {
+    select(title: string, options: string[]): Promise<string | undefined>;
+    editor(title: string, prefill?: string): Promise<string | undefined>;
+    notify(message: string, type?: "info" | "warning" | "error"): void;
+  };
 }
 
 export async function handleOfficialElicitation(
   request: DirectBrokerElicitationRequest,
-  ctx: { hasUI: boolean; ui: {
-    select(title: string, options: string[]): Promise<string | undefined>;
-    editor(title: string, prefill?: string): Promise<string | undefined>;
-    notify(message: string, type?: "info" | "warning" | "error"): void;
-  } },
+  ctx: PiElicitationContext,
   openUrl: (url: string) => Promise<boolean>,
 ): Promise<DirectBrokerElicitationResponse> {
   if (!ctx.hasUI) return { action: "cancel" };
-  const message = typeof request.message === "string" ? request.message : "Official Computer Use requests input";
+  const message = request.message ?? "Official Computer Use requests input";
 
   if (request.mode === "url") {
-    if (typeof request.url !== "string") return { action: "cancel" };
+    if (!request.url) return { action: "cancel" };
     const choice = await ctx.ui.select(`${message}\n${request.url}`, ["Open URL", "Decline", "Cancel"]);
     if (choice === "Decline") return { action: "decline" };
     if (choice !== "Open URL") return { action: "cancel" };
@@ -247,19 +103,21 @@ export async function handleOfficialElicitation(
     return { action: "cancel" };
   }
   const openAiForm = request.mode === "openai/form";
-  if (request.requestedSchema === undefined || (!openAiForm && !isRecord(request.requestedSchema))) return { action: "cancel" };
+  if (request.requestedSchema === undefined || (!openAiForm && !jsonObjectSchema.safeParse(request.requestedSchema).success)) {
+    return { action: "cancel" };
+  }
   const choice = await ctx.ui.select(message, ["Respond", "Decline", "Cancel"]);
   if (choice === "Decline") return { action: "decline" };
   if (choice !== "Respond") return { action: "cancel" };
-  let prefill = JSON.stringify(initialFormContent(request.requestedSchema), null, 2);
   const title = `${message}\nSchema: ${JSON.stringify(request.requestedSchema)}`;
+  let prefill = "{}";
   while (true) {
     const edited = await ctx.ui.editor(title, prefill);
     if (edited === undefined) return { action: "cancel" };
     prefill = edited;
     try {
-      const content = JSON.parse(edited);
-      if (!openAiForm && !isRecord(content)) throw new Error("Response must be a JSON object");
+      const parsed = JSON.parse(edited);
+      const content = openAiForm ? parsed : jsonObjectSchema.parse(parsed);
       return { action: "accept", content };
     } catch (error) {
       ctx.ui.notify(error instanceof Error ? error.message : "Response must be valid JSON", "error");
@@ -269,62 +127,59 @@ export async function handleOfficialElicitation(
 
 export default function directComputerUse(pi: ExtensionAPI) {
   const stateRoot = process.env.CODEX_COMPUTER_USE_HOME || path.join(getAgentDir(), "direct-computer-use");
-  const sessionExecutor = new PiDirectSessionExecutor();
+  const sessionExecutor = new DirectSessionExecutor();
 
   pi.registerCommand("computer-use-status", {
-    description: "Show the direct-call architecture, durable no-permissions policy, signed broker, and private audit path",
+    description: "Show Computer Use status",
     handler: async (_args, ctx) => {
-      const status = await getDirectStatus(stateRoot);
-      ctx.ui.notify(JSON.stringify(status, null, 2), "info");
+      ctx.ui.notify(JSON.stringify(getDirectStatus(stateRoot), null, 2), "info");
     },
   });
 
-  const methods = Object.keys(ToolParameters) as DirectMethod[];
-  for (const method of methods) {
+  for (const method of COMPUTER_USE_METHODS) {
     const piName = `computer_use_${method}`;
-    const inspectionPromptMetadata = INSPECTION_METHODS.has(method) ? {
-      promptSnippet: `${titleFor(method)} through the official signed macOS Computer Use service`,
-      promptGuidelines: [
-        `${piName} is a direct typed tool: choose its arguments yourself; it does not invoke a nested planner or model.`,
-        `Call computer_use_get_app_state once per assistant turn before interacting with an app.`,
-      ],
-    } : {};
     pi.registerTool({
       name: piName,
       label: titleFor(method),
-      description: toolDescription(method),
-      ...inspectionPromptMetadata,
-      parameters: ToolParameters[method] as any,
-      async execute(_toolCallId, params, signal, onUpdate, ctx) {
-        const response = await sessionExecutor.execute(
-          method,
-          params as Record<string, unknown>,
-          {
-            stateRoot,
-            signal,
-            supportsOpenAiFormElicitation: true,
-            onElicitation: (request) => handleOfficialElicitation(
-              request,
-              ctx,
-              async (url) => {
-                const result = await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 });
-                return result.code === 0;
-              },
-            ),
-            onProgress: (message) => onUpdate?.({ content: [{ type: "text", text: message }], details: { status: "running" } }),
-          },
-        );
-        if (response.isError) throw new Error(errorText(response.content));
-        return { content: toPiContent(response.content), details: response.details };
+      description: TOOL_METADATA[method].description,
+      // SAFETY: Pi accepts standard JSON Schema objects; these schemas are generated directly from the tool's Zod parser.
+      parameters: TOOL_INPUT_SCHEMAS[method] as any,
+      async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+        const parsedParams = jsonObjectSchema.parse(params);
+        const response = await sessionExecutor.execute(method, parsedParams, {
+          stateRoot,
+          signal,
+          supportsOpenAiFormElicitation: true,
+          onElicitation: (request) => handleOfficialElicitation(
+            request,
+            ctx,
+            async (url) => (await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 })).code === 0,
+          ),
+        });
+        if (response.isError) {
+          const message = response.content
+            .filter((block) => block.type === "text")
+            .map((block) => String(block.text ?? ""))
+            .join("\n")
+            .slice(0, 2_000);
+          throw new Error(message || "Official Computer Use returned an error");
+        }
+        const rendered = await toPiContent(response.content);
+        const details: JsonObject = {};
+        if (response.structuredContent !== undefined) details.officialStructuredContent = response.structuredContent;
+        if (rendered.fullOutputPath) details.fullOutputPath = rendered.fullOutputPath;
+        return { content: rendered.content, details };
       },
       renderCall(args, theme) {
-        const renderedArgs = args as Record<string, unknown>;
-        const app = typeof renderedArgs.app === "string" ? ` ${renderedArgs.app}` : "";
-        return new Text(theme.fg("toolTitle", theme.bold(`${piName} `)) + theme.fg("accent", app.trim()), 0, 0);
+        const parsed = z.object({ app: z.string().optional() }).safeParse(args);
+        const app = parsed.success ? parsed.data.app : undefined;
+        const label = theme.fg("toolTitle", theme.bold(piName));
+        return new Text(app ? `${label} ${theme.fg("accent", app)}` : label, 0, 0);
       },
     });
   }
 
-  pi.on("session_start", () => setInitialComputerUseTools(pi));
+  pi.on("session_start", () => pi.setActiveTools([...new Set([...pi.getActiveTools(), ...toolNames])]));
+  pi.on("agent_settled", () => sessionExecutor.close());
   pi.on("session_shutdown", () => sessionExecutor.close());
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-computer-use-mcp",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "description": "Expose the official macOS Computer Use tools directly to Pi and MCP clients without a nested model.",
   "author": "Thomas Mustier",
   "license": "MIT",
@@ -28,57 +28,44 @@
   },
   "files": [
     "dist",
-    "integrations/pi",
-    "npm-shrinkwrap.json",
+    "integrations/pi/index.ts",
     "README.md",
-    "CHANGELOG.md",
-    "PROOF.md",
-    "ARCHITECTURE.md",
-    "MIGRATION.md",
     "SECURITY.md",
-    "docs/adr",
-    "CONTRIBUTING.md",
     "LICENSE"
   ],
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.json",
-    "check": "tsc -p tsconfig.json --noEmit",
+    "lint": "oxlint --deny-warnings",
+    "check": "npm run lint && tsc -p tsconfig.json --noEmit",
+    "pretest": "npm run build",
     "test": "node --test test/*.test.ts",
-    "prepare": "npm run build",
-    "prepublishOnly": "npm run check && npm run check:pi && npm test && npm run build",
-    "check:pi": "npx tsc --ignoreConfig --noEmit --target ES2022 --module NodeNext --moduleResolution NodeNext --strict --skipLibCheck --types node integrations/pi/index.ts"
+    "prepack": "npm run build",
+    "prepublishOnly": "npm run check && npm run check:pi && npm test",
+    "check:pi": "npm run build && npx tsc --ignoreConfig --noEmit --target ES2022 --module NodeNext --moduleResolution NodeNext --strict --skipLibCheck --types node integrations/pi/index.ts"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "1.29.0",
-    "@types/node": "24.13.3",
-    "typescript": "7.0.2",
     "zod": "4.4.3"
   },
   "peerDependencies": {
-    "@earendil-works/pi-ai": "*",
     "@earendil-works/pi-coding-agent": "*",
-    "@earendil-works/pi-tui": "*",
-    "typebox": "*"
+    "@earendil-works/pi-tui": "*"
   },
   "peerDependenciesMeta": {
-    "@earendil-works/pi-ai": {
-      "optional": true
-    },
     "@earendil-works/pi-coding-agent": {
       "optional": true
     },
     "@earendil-works/pi-tui": {
       "optional": true
-    },
-    "typebox": {
-      "optional": true
     }
   },
   "devDependencies": {
-    "@earendil-works/pi-ai": "0.80.7",
     "@earendil-works/pi-coding-agent": "0.80.7",
     "@earendil-works/pi-tui": "0.80.7",
-    "typebox": "1.1.38"
+    "@oxlint/plugins": "1.79.0",
+    "@types/node": "24.13.3",
+    "oxlint": "1.79.0",
+    "typescript": "7.0.2"
   },
   "engines": {
     "node": ">=22"