codex-computer-use-mcp 0.3.4 → 0.5.0

package/integrations/pi/index.ts

@@ -1,237 +1,128 @@
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import path from "node:path";
-import { StringEnum } from "@earendil-works/pi-ai";
-import { getAgentDir, truncateHead, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import {
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+  formatSize,
+  getAgentDir,
+  truncateHead,
+  type ExtensionAPI,
+} from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
-import { Type, type TSchema } from "typebox";
-import { executeDirectTool, getDirectStatus } from "../../dist/direct-service.js";
+import { z } from "zod";
+import { getDirectStatus } from "../../dist/direct-service.js";
 import {
-  createOfficialDirectToolSession,
   type DirectBrokerElicitationRequest,
   type DirectBrokerElicitationResponse,
-  type OfficialDirectToolSession,
 } from "../../dist/direct-broker.js";
-import { OFFICIAL_TOOL_METADATA, type DirectMethod } from "../../dist/tools.js";
-
-const App = Type.String({ description: "App name, full app path, or unambiguous bundle identifier" });
-
-const ToolParameters: Record<DirectMethod, TSchema> = {
-  list_apps: Type.Object({}, { additionalProperties: false }),
-  get_app_state: Type.Object({ app: App }, { additionalProperties: false }),
-  click: Type.Object({
-    app: App,
-    click_count: Type.Optional(Type.Integer({ description: "Number of clicks. Defaults to 1" })),
-    element_index: Type.Optional(Type.String({ description: "Element index to click" })),
-    mouse_button: Type.Optional(StringEnum(["left", "right", "middle"] as const, { description: "Mouse button to click. Defaults to left." })),
-    x: Type.Optional(Type.Number({ description: "X coordinate in screenshot pixel coordinates" })),
-    y: Type.Optional(Type.Number({ description: "Y coordinate in screenshot pixel coordinates" })),
-  }, { additionalProperties: false }),
-  perform_secondary_action: Type.Object({
-    app: App,
-    element_index: Type.String({ description: "Element identifier" }),
-    action: Type.String({ description: "Secondary accessibility action name" }),
-  }, { additionalProperties: false }),
-  set_value: Type.Object({
-    app: App,
-    element_index: Type.String({ description: "Element identifier" }),
-    value: Type.String({ description: "Value to assign" }),
-  }, { additionalProperties: false }),
-  select_text: Type.Object({
-    app: Type.String({ description: "App name or bundle identifier" }),
-    element_index: Type.String({ description: "Text element identifier" }),
-    text: Type.String({ description: "Target text as shown in the accessibility tree" }),
-    prefix: Type.Optional(Type.String({ description: "Optional text immediately before the target, used to disambiguate repeated matches" })),
-    selection: Type.Optional(StringEnum(["text", "cursor_before", "cursor_after"] as const, { description: "Whether to select the text or place the cursor before or after it. Defaults to text." })),
-    suffix: Type.Optional(Type.String({ description: "Optional text immediately after the target, used to disambiguate repeated matches" })),
-  }, { additionalProperties: false }),
-  scroll: Type.Object({
-    app: App,
-    element_index: Type.String({ description: "Element identifier" }),
-    direction: Type.String({ description: "Scroll direction: up, down, left, or right" }),
-    pages: Type.Optional(Type.Number({ description: "Number of pages to scroll. Fractional values are supported. Defaults to 1" })),
-  }, { additionalProperties: false }),
-  drag: Type.Object({
-    app: App,
-    from_x: Type.Number({ description: "Start X coordinate" }),
-    from_y: Type.Number({ description: "Start Y coordinate" }),
-    to_x: Type.Number({ description: "End X coordinate" }),
-    to_y: Type.Number({ description: "End Y coordinate" }),
-  }, { additionalProperties: false }),
-  press_key: Type.Object({ app: App, key: Type.String({ description: "Key or key combination to press" }) }, { additionalProperties: false }),
-  type_text: Type.Object({ app: App, text: Type.String({ description: "Literal text to type" }) }, { additionalProperties: false }),
-};
-
-function toolDescription(method: DirectMethod): string {
-  return OFFICIAL_TOOL_METADATA[method].description;
-}
-
-function titleFor(method: DirectMethod): string {
-  return method.split("_").map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`).join(" ");
-}
-
-const INSPECTION_METHODS = new Set<DirectMethod>(["list_apps", "get_app_state"]);
-export const INSPECTION_TOOL_NAMES = [
-  "computer_use_list_apps",
-  "computer_use_get_app_state",
-] as const;
-export const INTERACTION_TOOL_NAMES = [
-  "computer_use_click",
-  "computer_use_perform_secondary_action",
-  "computer_use_set_value",
-  "computer_use_select_text",
-  "computer_use_scroll",
-  "computer_use_drag",
-  "computer_use_press_key",
-  "computer_use_type_text",
-] as const;
-
-export const COMPUTER_USE_TOOL_NAMES = [
-  ...INSPECTION_TOOL_NAMES,
-  ...INTERACTION_TOOL_NAMES,
-] as const;
-
-export function setInitialComputerUseTools(pi: Pick<ExtensionAPI, "getActiveTools" | "setActiveTools">): void {
-  pi.setActiveTools([...new Set([...pi.getActiveTools(), ...COMPUTER_USE_TOOL_NAMES])]);
-}
-
-interface SessionExecutorDependencies {
-  createSession?: typeof createOfficialDirectToolSession;
-  executeTool?: typeof executeDirectTool;
+import { ComputerUseCodeExecutor } from "../../dist/code-executor.js";
+import { type JsonObject } from "../../dist/tools.js";
+
+const jsonObjectSchema = z.record(z.string(), z.json());
+const codeParameters = {
+  type: "object",
+  additionalProperties: false,
+  required: ["code"],
+  properties: {
+    code: {
+      type: "string",
+      description: "JavaScript body to execute. Use await sky.<method>(args), emit(value), emitImage(screenshot), and store for state shared across calls.",
+    },
+  },
+} as const;
+
+const codeDescription = `Run JavaScript that composes OpenAI's official signed macOS Computer Use methods in one call. No nested model is used.
+
+Available globals:
+- sky.list_apps() -> text app inventory
+- sky.get_app_state({ app, disableDiff? }) -> { app, text, screenshot }
+- sky.click({ app, element_index?, x?, y?, mouse_button?, click_count? })
+- sky.perform_secondary_action({ app, element_index, action })
+- sky.set_value({ app, element_index, value })
+- sky.select_text({ app, element_index, text, prefix?, suffix?, selection? })
+- sky.scroll({ app, element_index, direction, pages? })
+- sky.drag({ app, from_x, from_y, to_x, to_y })
+- sky.press_key({ app, key })
+- sky.type_text({ app, text })
+- emit(value) returns text or JSON to Pi
+- emitImage(state.screenshot) returns a screenshot to Pi
+- store is a persistent JSON object shared across calls
+
+get_app_state may return an accessibility-tree diff after the first inspection. Pass disableDiff: true when you need a fresh full tree.
+
+Example:
+const state = await sky.get_app_state({ app: "TextEdit" });
+emit(state.text);
+
+Batch known actions sequentially, then inspect again before deciding the next step.`;
+
+interface PiContentResult {
+  content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }>;
+  fullOutputPath?: string;
 }
 
-/** Keep the signed app-server/client alive across the required state -> action pair. */
-export class PiDirectSessionExecutor {
-  private session: OfficialDirectToolSession | undefined;
-  private queue: Promise<void> = Promise.resolve();
-  private readonly dependencies: SessionExecutorDependencies;
-
-  constructor(dependencies: SessionExecutorDependencies = {}) {
-    this.dependencies = dependencies;
-  }
-
-  private async closeSession(): Promise<void> {
-    const session = this.session;
-    this.session = undefined;
-    await session?.close();
+export async function toPiContent(content: JsonObject[]): Promise<PiContentResult> {
+  const textBlocks = content.filter((block) => block.type !== "image").map((block) => String(block.text ?? ""));
+  const fullText = textBlocks.join("\n\n");
+  const aggregate = truncateHead(fullText, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
+  if (!aggregate.truncated) {
+    return {
+      content: content.map((block) => block.type === "image"
+        ? { type: "image", data: String(block.data), mimeType: String(block.mimeType) }
+        : { type: "text", text: String(block.text ?? "") }),
+    };
   }
 
-  async execute(
-    method: DirectMethod,
-    params: Record<string, unknown>,
-    dependencies: Parameters<typeof executeDirectTool>[1],
-  ): Promise<Awaited<ReturnType<typeof executeDirectTool>>> {
-    const previous = this.queue;
-    let release!: () => void;
-    this.queue = new Promise<void>((resolve) => { release = resolve; });
-    await previous;
-    try {
-      const executeTool = this.dependencies.executeTool ?? executeDirectTool;
-      if (method === "list_apps") return await executeTool({ method, arguments: params }, dependencies);
-      if (method === "get_app_state") {
-        await this.closeSession();
-        const createSession = this.dependencies.createSession ?? createOfficialDirectToolSession;
-        try {
-          const response = await executeTool(
-            { method, arguments: params },
-            {
-              ...dependencies,
-              callTool: async (directMethod, args, options) => {
-                this.session = await createSession({ supportsOpenAiFormElicitation: true });
-                return this.session.call(directMethod, args, options);
-              },
-            },
-          );
-          if (response.isError) await this.closeSession();
-          return response;
-        } catch (error) {
-          await this.closeSession();
-          throw error;
-        }
-      }
-      const session = this.session;
-      try {
-        return await executeTool(
-          { method, arguments: params },
-          session ? {
-            ...dependencies,
-            callTool: async (directMethod, args, options) => {
-              const result = await session.call(directMethod, args, options);
-              await this.closeSession();
-              return { ...result, brokerCleanupVerified: true };
-            },
-          } : dependencies,
-        );
-      } finally {
-        await this.closeSession();
-      }
-    } finally {
-      release();
-    }
-  }
-
-  async close(): Promise<void> {
-    await this.queue;
-    await this.closeSession();
-  }
-}
-
-function toPiContent(content: Array<Record<string, unknown>>): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
-  const result: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = [];
+  const tempDir = await mkdtemp(path.join(tmpdir(), "pi-computer-use-"));
+  const fullOutputPath = path.join(tempDir, "output.txt");
+  await writeFile(fullOutputPath, fullText, { encoding: "utf8", mode: 0o600 });
+  const suffix = `\n\n[Official Computer Use text truncated: showing ${aggregate.outputLines} of ${aggregate.totalLines} lines (${formatSize(aggregate.outputBytes)} of ${formatSize(aggregate.totalBytes)}). Full output saved to: ${fullOutputPath}]`;
+  const result: PiContentResult = { content: [], fullOutputPath };
+  let textIndex = 0;
+  let textOffset = 0;
+  let noticeAdded = false;
   for (const block of content) {
     if (block.type === "image") {
-      result.push({ type: "image", data: String(block.data), mimeType: String(block.mimeType) });
+      result.content.push({ type: "image", data: String(block.data), mimeType: String(block.mimeType) });
       continue;
     }
-    const original = String(block.text ?? "");
-    const truncated = truncateHead(original, { maxLines: 2_000, maxBytes: 50 * 1024 });
-    const suffix = truncated.truncated
-      ? `\n\n[Official Computer Use text truncated in-memory: ${truncated.outputLines}/${truncated.totalLines} lines, ${truncated.outputBytes}/${truncated.totalBytes} bytes. No full-output file was written.]`
+    const blockText = String(block.text ?? "");
+    const start = textOffset + (textIndex > 0 ? 2 : 0);
+    const end = start + blockText.length;
+    const retained = aggregate.content.length > start
+      ? blockText.slice(0, Math.min(end, aggregate.content.length) - start)
       : "";
-    result.push({ type: "text", text: `${truncated.content}${suffix}` });
+    const cutoffHere: boolean = !noticeAdded && aggregate.content.length < end;
+    if (retained || cutoffHere) {
+      result.content.push({ type: "text", text: `${retained}${cutoffHere ? suffix : ""}` });
+      noticeAdded ||= cutoffHere;
+    }
+    textOffset = end;
+    textIndex += 1;
   }
   return result;
 }
 
-function errorText(content: Array<Record<string, unknown>>): string {
-  return content
-    .filter((block) => block.type === "text")
-    .map((block) => String(block.text ?? ""))
-    .join("\n")
-    .slice(0, 2_000) || "Official Computer Use returned an error";
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
-}
-
-function initialFormContent(schema: unknown): Record<string, unknown> {
-  if (!isRecord(schema) || !isRecord(schema.properties)) return {};
-  const content: Record<string, unknown> = {};
-  for (const [key, rawProperty] of Object.entries(schema.properties)) {
-    if (!isRecord(rawProperty)) continue;
-    if (rawProperty.default !== undefined) content[key] = rawProperty.default;
-    else if (Array.isArray(rawProperty.enum) && rawProperty.enum.length > 0) content[key] = rawProperty.enum[0];
-    else if (rawProperty.type === "boolean") content[key] = false;
-    else if (rawProperty.type === "number" || rawProperty.type === "integer") content[key] = 0;
-    else if (rawProperty.type === "array") content[key] = [];
-    else content[key] = "";
-  }
-  return content;
+interface PiElicitationContext {
+  hasUI: boolean;
+  ui: {
+    select(title: string, options: string[]): Promise<string | undefined>;
+    editor(title: string, prefill?: string): Promise<string | undefined>;
+    notify(message: string, type?: "info" | "warning" | "error"): void;
+  };
 }
 
 export async function handleOfficialElicitation(
   request: DirectBrokerElicitationRequest,
-  ctx: { hasUI: boolean; ui: {
-    select(title: string, options: string[]): Promise<string | undefined>;
-    editor(title: string, prefill?: string): Promise<string | undefined>;
-    notify(message: string, type?: "info" | "warning" | "error"): void;
-  } },
+  ctx: PiElicitationContext,
   openUrl: (url: string) => Promise<boolean>,
 ): Promise<DirectBrokerElicitationResponse> {
   if (!ctx.hasUI) return { action: "cancel" };
-  const message = typeof request.message === "string" ? request.message : "Official Computer Use requests input";
+  const message = request.message ?? "Official Computer Use requests input";
 
   if (request.mode === "url") {
-    if (typeof request.url !== "string") return { action: "cancel" };
+    if (!request.url) return { action: "cancel" };
     const choice = await ctx.ui.select(`${message}\n${request.url}`, ["Open URL", "Decline", "Cancel"]);
     if (choice === "Decline") return { action: "decline" };
     if (choice !== "Open URL") return { action: "cancel" };
@@ -247,19 +138,21 @@ export async function handleOfficialElicitation(
     return { action: "cancel" };
   }
   const openAiForm = request.mode === "openai/form";
-  if (request.requestedSchema === undefined || (!openAiForm && !isRecord(request.requestedSchema))) return { action: "cancel" };
+  if (request.requestedSchema === undefined || (!openAiForm && !jsonObjectSchema.safeParse(request.requestedSchema).success)) {
+    return { action: "cancel" };
+  }
   const choice = await ctx.ui.select(message, ["Respond", "Decline", "Cancel"]);
   if (choice === "Decline") return { action: "decline" };
   if (choice !== "Respond") return { action: "cancel" };
-  let prefill = JSON.stringify(initialFormContent(request.requestedSchema), null, 2);
   const title = `${message}\nSchema: ${JSON.stringify(request.requestedSchema)}`;
+  let prefill = "{}";
   while (true) {
     const edited = await ctx.ui.editor(title, prefill);
     if (edited === undefined) return { action: "cancel" };
     prefill = edited;
     try {
-      const content = JSON.parse(edited);
-      if (!openAiForm && !isRecord(content)) throw new Error("Response must be a JSON object");
+      const parsed = JSON.parse(edited);
+      const content = openAiForm ? parsed : jsonObjectSchema.parse(parsed);
       return { action: "accept", content };
     } catch (error) {
       ctx.ui.notify(error instanceof Error ? error.message : "Response must be valid JSON", "error");
@@ -269,62 +162,54 @@ export async function handleOfficialElicitation(
 
 export default function directComputerUse(pi: ExtensionAPI) {
   const stateRoot = process.env.CODEX_COMPUTER_USE_HOME || path.join(getAgentDir(), "direct-computer-use");
-  const sessionExecutor = new PiDirectSessionExecutor();
+  const codeExecutor = new ComputerUseCodeExecutor();
 
   pi.registerCommand("computer-use-status", {
-    description: "Show the direct-call architecture, durable no-permissions policy, signed broker, and private audit path",
+    description: "Show Computer Use status",
     handler: async (_args, ctx) => {
-      const status = await getDirectStatus(stateRoot);
-      ctx.ui.notify(JSON.stringify(status, null, 2), "info");
+      ctx.ui.notify(JSON.stringify(getDirectStatus(stateRoot), null, 2), "info");
     },
   });
 
-  const methods = Object.keys(ToolParameters) as DirectMethod[];
-  for (const method of methods) {
-    const piName = `computer_use_${method}`;
-    const inspectionPromptMetadata = INSPECTION_METHODS.has(method) ? {
-      promptSnippet: `${titleFor(method)} through the official signed macOS Computer Use service`,
-      promptGuidelines: [
-        `${piName} is a direct typed tool: choose its arguments yourself; it does not invoke a nested planner or model.`,
-        `Call computer_use_get_app_state once per assistant turn before interacting with an app.`,
-      ],
-    } : {};
-    pi.registerTool({
-      name: piName,
-      label: titleFor(method),
-      description: toolDescription(method),
-      ...inspectionPromptMetadata,
-      parameters: ToolParameters[method] as any,
-      async execute(_toolCallId, params, signal, onUpdate, ctx) {
-        const response = await sessionExecutor.execute(
-          method,
-          params as Record<string, unknown>,
-          {
-            stateRoot,
-            signal,
-            supportsOpenAiFormElicitation: true,
-            onElicitation: (request) => handleOfficialElicitation(
-              request,
-              ctx,
-              async (url) => {
-                const result = await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 });
-                return result.code === 0;
-              },
-            ),
-            onProgress: (message) => onUpdate?.({ content: [{ type: "text", text: message }], details: { status: "running" } }),
-          },
-        );
-        if (response.isError) throw new Error(errorText(response.content));
-        return { content: toPiContent(response.content), details: response.details };
-      },
-      renderCall(args, theme) {
-        const renderedArgs = args as Record<string, unknown>;
-        const app = typeof renderedArgs.app === "string" ? ` ${renderedArgs.app}` : "";
-        return new Text(theme.fg("toolTitle", theme.bold(`${piName} `)) + theme.fg("accent", app.trim()), 0, 0);
-      },
-    });
-  }
+  pi.registerTool({
+    name: "computer_use",
+    label: "Computer Use",
+    description: codeDescription,
+    promptSnippet: "Run composable JavaScript against OpenAI's official signed macOS Computer Use surface",
+    promptGuidelines: [
+      "Use computer_use for macOS app UI work, composing known sequential actions in one JavaScript call and emitting only the state needed for the next decision.",
+    ],
+    // SAFETY: Pi accepts this standard JSON Schema object as a custom-tool parameter schema.
+    parameters: codeParameters as any,
+    async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+      const { code } = z.object({ code: z.string() }).parse(params);
+      const result = await codeExecutor.execute(code, {
+        stateRoot,
+        signal,
+        supportsOpenAiFormElicitation: true,
+        onElicitation: (request) => handleOfficialElicitation(
+          request,
+          ctx,
+          async (url) => (await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 })).code === 0,
+        ),
+      });
+      const rendered = await toPiContent(result.content);
+      const details: JsonObject = { calls: result.calls };
+      if (result.error) details.error = result.error;
+      if (rendered.fullOutputPath) details.fullOutputPath = rendered.fullOutputPath;
+      return { content: rendered.content, details };
+    },
+    renderCall(args, theme) {
+      const parsed = z.object({ code: z.string() }).safeParse(args);
+      const firstLine = parsed.success
+        ? parsed.data.code.split("\n").map((line) => line.trim()).find(Boolean)?.slice(0, 100)
+        : undefined;
+      const label = theme.fg("toolTitle", theme.bold("computer_use"));
+      return new Text(firstLine ? `${label} ${theme.fg("dim", firstLine)}` : label, 0, 0);
+    },
+  });
 
-  pi.on("session_start", () => setInitialComputerUseTools(pi));
-  pi.on("session_shutdown", () => sessionExecutor.close());
+  pi.on("session_start", () => pi.setActiveTools([...new Set([...pi.getActiveTools(), "computer_use"])]));
+  pi.on("agent_settled", () => codeExecutor.close());
+  pi.on("session_shutdown", () => codeExecutor.close());
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-computer-use-mcp",
-  "version": "0.3.4",
+  "version": "0.5.0",
   "description": "Expose the official macOS Computer Use tools directly to Pi and MCP clients without a nested model.",
   "author": "Thomas Mustier",
   "license": "MIT",
@@ -28,57 +28,44 @@
   },
   "files": [
     "dist",
-    "integrations/pi",
-    "npm-shrinkwrap.json",
+    "integrations/pi/index.ts",
     "README.md",
-    "CHANGELOG.md",
-    "PROOF.md",
-    "ARCHITECTURE.md",
-    "MIGRATION.md",
     "SECURITY.md",
-    "docs/adr",
-    "CONTRIBUTING.md",
     "LICENSE"
   ],
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.json",
-    "check": "tsc -p tsconfig.json --noEmit",
+    "lint": "oxlint --deny-warnings",
+    "check": "npm run lint && tsc -p tsconfig.json --noEmit",
+    "pretest": "npm run build",
     "test": "node --test test/*.test.ts",
-    "prepare": "npm run build",
-    "prepublishOnly": "npm run check && npm run check:pi && npm test && npm run build",
-    "check:pi": "npx tsc --ignoreConfig --noEmit --target ES2022 --module NodeNext --moduleResolution NodeNext --strict --skipLibCheck --types node integrations/pi/index.ts"
+    "prepack": "npm run build",
+    "prepublishOnly": "npm run check && npm run check:pi && npm test",
+    "check:pi": "npm run build && npx tsc --ignoreConfig --noEmit --target ES2022 --module NodeNext --moduleResolution NodeNext --strict --skipLibCheck --types node integrations/pi/index.ts"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "1.29.0",
-    "@types/node": "24.13.3",
-    "typescript": "7.0.2",
     "zod": "4.4.3"
   },
   "peerDependencies": {
-    "@earendil-works/pi-ai": "*",
     "@earendil-works/pi-coding-agent": "*",
-    "@earendil-works/pi-tui": "*",
-    "typebox": "*"
+    "@earendil-works/pi-tui": "*"
   },
   "peerDependenciesMeta": {
-    "@earendil-works/pi-ai": {
-      "optional": true
-    },
     "@earendil-works/pi-coding-agent": {
       "optional": true
     },
     "@earendil-works/pi-tui": {
       "optional": true
-    },
-    "typebox": {
-      "optional": true
     }
   },
   "devDependencies": {
-    "@earendil-works/pi-ai": "0.80.7",
     "@earendil-works/pi-coding-agent": "0.80.7",
     "@earendil-works/pi-tui": "0.80.7",
-    "typebox": "1.1.38"
+    "@oxlint/plugins": "1.79.0",
+    "@types/node": "24.13.3",
+    "oxlint": "1.79.0",
+    "typescript": "7.0.2"
   },
   "engines": {
     "node": ">=22"