codex-computer-use-mcp 0.4.0 → 0.5.0

package/README.md

@@ -22,21 +22,26 @@ Install from npm:
 pi install npm:codex-computer-use-mcp
 ```
 
-The extension registers and activates these tools:
+The extension registers and activates one composable tool:
 
 ```text
-computer_use_list_apps
-computer_use_get_app_state
-computer_use_click
-computer_use_perform_secondary_action
-computer_use_set_value
-computer_use_select_text
-computer_use_scroll
-computer_use_drag
-computer_use_press_key
-computer_use_type_text
+computer_use({ code: string })
 ```
 
+The code runs with `sky`, `emit`, `emitImage` and a persistent `store` object. `sky` exposes all ten official Computer Use methods, so known sequential actions can run without a model round-trip between each action:
+
+```js
+const state = await sky.get_app_state({ app: "TextEdit" });
+emit(state.text);
+
+await sky.click({ app: "TextEdit", element_index: "7" });
+await sky.type_text({ app: "TextEdit", text: "hello" });
+const next = await sky.get_app_state({ app: "TextEdit" });
+emit(next.text);
+```
+
+Only values passed to `emit(...)` or `emitImage(...)` are returned to Pi. `list_apps` returns the official text inventory produced by the app-server transport; unlike native `@oai/sky`, that transport does not provide the structured app array. `get_app_state` may return an accessibility-tree diff after the first inspection; pass `disableDiff: true` to request a fresh full tree. Screenshot payloads remain in the parent process and cross the code-worker boundary only as small opaque handles. Code runs in a worker so an unbounded loop can be terminated without freezing Pi; time spent inside an official Computer Use call does not count toward the code execution slice. If a later action fails, Pi still receives earlier emitted observations and the attempted method sequence.
+
 Use `/computer-use-status` to inspect the installed component and transport status.
 
 To run a source checkout:
@@ -73,7 +78,7 @@ Example Pi MCP configuration:
 
 ## Behaviour
 
-The adapter has one mode. All ten official methods are available without wrapper permission prompts. It adds no app allowlist, action gate, intent classifier, selector rewrite or focus policy. App-server uses Codex Full access. The adapter forwards any elicitation that the official host still emits.
+The adapter has one mode. Pi exposes the ten official methods through the single `computer_use` code tool; MCP exposes them as ten typed methods. Both are available without wrapper permission prompts. It adds no app allowlist, action gate, intent classifier, selector rewrite or focus policy. App-server uses Codex Full access. The adapter forwards any elicitation that the official host still emits.
 
 Production calls require verified OpenAI-signed app-server and Computer Use binaries with Team ID `2DC432GLL2`. The adapter uses an isolated, credential-free app-server context. It rejects any model-turn activity. Calls after `get_app_state` reuse the signed session, preserving element identifiers and official app state.
 

package/dist/code-executor.d.ts

@@ -0,0 +1,23 @@
+import type { DirectServiceDependencies } from "./direct-service.ts";
+import { DirectSessionExecutor } from "./session-executor.ts";
+import { type DirectMethod, type JsonObject, type JsonValue } from "./tools.ts";
+export interface ComputerUseCodeResult {
+    content: JsonObject[];
+    calls: DirectMethod[];
+    error?: string;
+}
+type CodeSessionExecutor = Pick<DirectSessionExecutor, "execute" | "close">;
+export declare class ComputerUseCodeExecutor {
+    private queue;
+    private readonly sessionExecutor;
+    private readonly codeSliceTimeoutMs;
+    private readonly screenshots;
+    private nextScreenshotId;
+    readonly store: Record<string, JsonValue | undefined>;
+    constructor(sessionExecutor?: CodeSessionExecutor, codeSliceTimeoutMs?: number);
+    private runExclusive;
+    private registerScreenshot;
+    execute(code: string, dependencies: DirectServiceDependencies): Promise<ComputerUseCodeResult>;
+    close(): Promise<void>;
+}
+export {};

package/dist/code-executor.js

@@ -0,0 +1,216 @@
+import { Worker } from "node:worker_threads";
+import { z } from "zod";
+import { DirectSessionExecutor } from "./session-executor.js";
+import { COMPUTER_USE_METHODS, } from "./tools.js";
+const MAX_CODE_BYTES = 20_000;
+const MAX_CALLS = 50;
+const MAX_SCREENSHOT_HANDLES = 50;
+const MAX_EMITTED_IMAGES = 10;
+const MAX_EMITTED_IMAGE_BYTES = 20 * 1024 * 1024;
+const CODE_SLICE_TIMEOUT_MS = 5_000;
+const WORKER_STARTUP_TIMEOUT_MS = 5_000;
+const workerMessageSchema = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("ready") }),
+    z.object({ type: z.literal("call"), id: z.number().int(), method: z.enum(COMPUTER_USE_METHODS), args: z.string() }),
+    z.object({ type: z.literal("emit"), value: z.string() }),
+    z.object({ type: z.literal("emit_image"), value: z.string() }),
+    z.object({ type: z.literal("done"), store: z.string(), error: z.string().optional() }),
+]);
+const argumentsSchema = z.record(z.string(), z.json());
+const emittedValueSchema = z.json();
+const emittedStringSchema = z.string();
+const screenshotHandleSchema = z.object({
+    type: z.literal("computer_use_screenshot"),
+    id: z.string(),
+});
+export class ComputerUseCodeExecutor {
+    queue = Promise.resolve();
+    sessionExecutor;
+    codeSliceTimeoutMs;
+    screenshots = new Map();
+    nextScreenshotId = 1;
+    store = {};
+    constructor(sessionExecutor = new DirectSessionExecutor(), codeSliceTimeoutMs = CODE_SLICE_TIMEOUT_MS) {
+        this.sessionExecutor = sessionExecutor;
+        this.codeSliceTimeoutMs = codeSliceTimeoutMs;
+    }
+    runExclusive(operation) {
+        const result = this.queue.then(operation);
+        this.queue = result.then(() => undefined, () => undefined);
+        return result;
+    }
+    registerScreenshot(image) {
+        const id = String(this.nextScreenshotId++);
+        this.screenshots.set(id, image);
+        if (this.screenshots.size > MAX_SCREENSHOT_HANDLES) {
+            this.screenshots.delete(this.screenshots.keys().next().value);
+        }
+        return { type: "computer_use_screenshot", id };
+    }
+    execute(code, dependencies) {
+        return this.runExclusive(async () => {
+            if (Buffer.byteLength(code, "utf8") > MAX_CODE_BYTES) {
+                throw new Error(`Computer Use code exceeds ${MAX_CODE_BYTES} bytes`);
+            }
+            return new Promise((resolve, reject) => {
+                const content = [];
+                const calls = [];
+                const worker = new Worker(import.meta.url.endsWith(".ts")
+                    ? new URL("../dist/code-worker.js", import.meta.url)
+                    : new URL("./code-worker.js", import.meta.url), { workerData: { code, store: this.store } });
+                let emittedImages = 0;
+                let emittedImageBytes = 0;
+                let settled = false;
+                let timer;
+                const clearTimer = () => {
+                    if (timer)
+                        clearTimeout(timer);
+                    timer = undefined;
+                };
+                const finish = (result) => {
+                    if (settled)
+                        return;
+                    settled = true;
+                    clearTimer();
+                    dependencies.signal?.removeEventListener("abort", abort);
+                    void worker.terminate();
+                    resolve(result);
+                };
+                const fail = (error) => {
+                    if (settled)
+                        return;
+                    settled = true;
+                    clearTimer();
+                    dependencies.signal?.removeEventListener("abort", abort);
+                    void worker.terminate();
+                    reject(error);
+                };
+                const stopWithError = (message) => {
+                    content.push({ type: "text", text: `Computer Use code stopped: ${message}` });
+                    finish({ content, calls, error: message });
+                };
+                const armTimer = () => {
+                    clearTimer();
+                    if (settled)
+                        return;
+                    timer = setTimeout(() => stopWithError(`execution exceeded ${this.codeSliceTimeoutMs}ms between Computer Use calls`), this.codeSliceTimeoutMs);
+                    timer.unref();
+                };
+                const abort = () => fail(new Error("Computer Use code cancelled"));
+                if (dependencies.signal?.aborted) {
+                    abort();
+                    return;
+                }
+                dependencies.signal?.addEventListener("abort", abort, { once: true });
+                let messageQueue = Promise.resolve();
+                worker.on("message", (rawMessage) => {
+                    messageQueue = messageQueue.then(async () => {
+                        if (settled)
+                            return;
+                        const message = workerMessageSchema.parse(rawMessage);
+                        if (message.type === "ready") {
+                            armTimer();
+                            return;
+                        }
+                        if (message.type === "emit") {
+                            const parsed = emittedValueSchema.parse(JSON.parse(message.value));
+                            const stringResult = emittedStringSchema.safeParse(parsed);
+                            content.push({ type: "text", text: stringResult.success ? stringResult.data : JSON.stringify(parsed, null, 2) });
+                            return;
+                        }
+                        if (message.type === "emit_image") {
+                            const handle = screenshotHandleSchema.parse(JSON.parse(message.value));
+                            const image = this.screenshots.get(handle.id);
+                            if (!image) {
+                                stopWithError("screenshot is no longer available");
+                                return;
+                            }
+                            const imageBytes = Buffer.byteLength(image.data, "utf8");
+                            if (emittedImages >= MAX_EMITTED_IMAGES || emittedImageBytes + imageBytes > MAX_EMITTED_IMAGE_BYTES) {
+                                stopWithError(`emitted images exceed ${MAX_EMITTED_IMAGES} images or ${MAX_EMITTED_IMAGE_BYTES} bytes`);
+                                return;
+                            }
+                            emittedImages += 1;
+                            emittedImageBytes += imageBytes;
+                            content.push(image);
+                            return;
+                        }
+                        if (message.type === "done") {
+                            const nextStore = argumentsSchema.parse(JSON.parse(message.store));
+                            for (const key of Object.keys(this.store))
+                                delete this.store[key];
+                            Object.assign(this.store, nextStore);
+                            if (message.error)
+                                stopWithError(message.error);
+                            else
+                                finish({ content, calls });
+                            return;
+                        }
+                        if (calls.length >= MAX_CALLS) {
+                            worker.postMessage({ type: "call_result", id: message.id, error: `Computer Use code exceeded ${MAX_CALLS} calls` });
+                            return;
+                        }
+                        clearTimer();
+                        calls.push(message.method);
+                        try {
+                            const args = argumentsSchema.parse(JSON.parse(message.args));
+                            const response = await this.sessionExecutor.execute(message.method, args, dependencies);
+                            const text = response.content
+                                .filter((block) => block.type !== "image")
+                                .map((block) => String(block.text ?? ""))
+                                .join("\n");
+                            if (response.isError)
+                                throw new Error(text.slice(0, 2_000) || "Official Computer Use returned an error");
+                            let value = response.structuredContent;
+                            if (message.method === "get_app_state") {
+                                const block = response.content.find((item) => item.type === "image");
+                                const screenshot = block ? {
+                                    type: "image",
+                                    data: String(block.data),
+                                    mimeType: String(block.mimeType),
+                                } : undefined;
+                                value = {
+                                    app: String(args.app),
+                                    text,
+                                    screenshot: screenshot ? this.registerScreenshot(screenshot) : null,
+                                };
+                            }
+                            else if (value === undefined) {
+                                value = text || undefined;
+                            }
+                            worker.postMessage({ type: "call_result", id: message.id, value: JSON.stringify(value ?? null) });
+                        }
+                        catch (error) {
+                            worker.postMessage({
+                                type: "call_result",
+                                id: message.id,
+                                error: error instanceof Error ? error.message : String(error),
+                            });
+                        }
+                        finally {
+                            armTimer();
+                        }
+                    }).catch((error) => fail(error instanceof Error ? error : new Error(String(error))));
+                });
+                worker.once("error", (error) => fail(error));
+                worker.once("exit", (code) => {
+                    if (!settled)
+                        fail(new Error(`Computer Use code worker exited before completion (${code})`));
+                });
+                timer = setTimeout(() => fail(new Error("Computer Use code worker failed to start")), WORKER_STARTUP_TIMEOUT_MS);
+                timer.unref();
+            });
+        });
+    }
+    async close() {
+        await this.runExclusive(async () => {
+            try {
+                await this.sessionExecutor.close();
+            }
+            finally {
+                this.screenshots.clear();
+            }
+        });
+    }
+}
+//# sourceMappingURL=code-executor.js.map

package/dist/code-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-executor.js","sourceRoot":"","sources":["../src/code-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EACN,oBAAoB,GAIpB,MAAM,YAAY,CAAC;AAEpB,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACjD,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAcxC,MAAM,mBAAmB,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACxD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;IACtC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnH,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC9D,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACtF,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACvD,MAAM,kBAAkB,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACpC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;AACvC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,yBAAyB,CAAC;IAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CACd,CAAC,CAAC;AAIH,MAAM,OAAO,uBAAuB;IAC3B,KAAK,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACjB,eAAe,CAAsB;IACrC,kBAAkB,CAAS;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IACrD,gBAAgB,GAAG,CAAC,CAAC;IACpB,KAAK,GAA0C,EAAE,CAAC;IAE3D,YAAY,eAAe,GAAwB,IAAI,qBAAqB,EAAE,EAAE,kBAAkB,GAAG,qBAAqB;QACzH,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC9C,CAAC;IAEO,YAAY,CAAI,SAA2B;QAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,kBAAkB,CAAC,KAAiB;QAC3C,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,sBAAsB,EAAE,CAAC;YACpD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,yBAAyB,EAAE,EAAE,EAAE,CAAC;IAChD,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,YAAuC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,cAAc,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CAAC,6BAA6B,cAAc,QAAQ,CAAC,CAAC;YACtE,CAAC;YACD,OAAO,IAAI,OAAO,CAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC7D,MAAM,OAAO,GAAiB,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAmB,EAAE,CAAC;gBACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CACxB,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC9B,CAAC,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC;oBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,kBAAkB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,EAC/C,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAC3C,CAAC;gBACF,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1B,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,IAAI,KAAiC,CAAC;gBACtC,MAAM,UAAU,GAAG,GAAS,EAAE;oBAC7B,IAAI,KAAK;wBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC/B,KAAK,GAAG,SAAS,CAAC;gBACnB,CAAC,CAAC;gBACF,MAAM,MAAM,GAAG,CAAC,MAA6B,EAAQ,EAAE;oBACtD,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,UAAU,EAAE,CAAC;oBACb,YAAY,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACzD,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;oBACxB,OAAO,CAAC,MAAM,CAAC,CAAC;gBACjB,CAAC,CAAC;gBACF,MAAM,IAAI,GAAG,CAAC,KAAY,EAAQ,EAAE;oBACnC,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,UAAU,EAAE,CAAC;oBACb,YAAY,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACzD,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;oBACxB,MAAM,CAAC,KAAK,CAAC,CAAC;gBACf,CAAC,CAAC;gBACF,MAAM,aAAa,GAAG,CAAC,OAAe,EAAQ,EAAE;oBAC/C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,8BAA8B,OAAO,EAAE,EAAE,CAAC,CAAC;oBAC9E,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5C,CAAC,CAAC;gBACF,MAAM,QAAQ,GAAG,GAAS,EAAE;oBAC3B,UAAU,EAAE,CAAC;oBACb,IAAI,OAAO;wBAAE,OAAO;oBACpB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,sBAAsB,IAAI,CAAC,kBAAkB,+BAA+B,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAC/I,KAAK,CAAC,KAAK,EAAE,CAAC;gBACf,CAAC,CAAC;gBACF,MAAM,KAAK,GAAG,GAAS,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;gBACzE,IAAI,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBAClC,KAAK,EAAE,CAAC;oBACR,OAAO;gBACR,CAAC;gBACD,YAAY,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtE,IAAI,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,UAA8B,EAAE,EAAE;oBACvD,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;wBAC3C,IAAI,OAAO;4BAAE,OAAO;wBACpB,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;wBACtD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;4BAC9B,QAAQ,EAAE,CAAC;4BACX,OAAO;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;4BAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;4BACnE,MAAM,YAAY,GAAG,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BAC3D,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;4BACjH,OAAO;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;4BACnC,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;4BACvE,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;4BAC9C,IAAI,CAAC,KAAK,EAAE,CAAC;gCACZ,aAAa,CAAC,mCAAmC,CAAC,CAAC;gCACnD,OAAO;4BACR,CAAC;4BACD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;4BACzD,IAAI,aAAa,IAAI,kBAAkB,IAAI,iBAAiB,GAAG,UAAU,GAAG,uBAAuB,EAAE,CAAC;gCACrG,aAAa,CAAC,yBAAyB,kBAAkB,cAAc,uBAAuB,QAAQ,CAAC,CAAC;gCACxG,OAAO;4BACR,CAAC;4BACD,aAAa,IAAI,CAAC,CAAC;4BACnB,iBAAiB,IAAI,UAAU,CAAC;4BAChC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;4BACpB,OAAO;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;4BAC7B,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;4BACnE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gCAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;4BAClE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;4BACrC,IAAI,OAAO,CAAC,KAAK;gCAAE,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;;gCAC3C,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;4BAChC,OAAO;wBACR,CAAC;wBAED,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;4BAC/B,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,8BAA8B,SAAS,QAAQ,EAAE,CAAC,CAAC;4BACpH,OAAO;wBACR,CAAC;wBACD,UAAU,EAAE,CAAC;wBACb,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC3B,IAAI,CAAC;4BACJ,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC7D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;4BACxF,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO;iCAC3B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;iCACzC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;iCACxC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACb,IAAI,QAAQ,CAAC,OAAO;gCAAE,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,yCAAyC,CAAC,CAAC;4BACzG,IAAI,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC;4BACvC,IAAI,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;gCACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;gCACrE,MAAM,UAAU,GAA2B,KAAK,CAAC,CAAC,CAAC;oCAClD,IAAI,EAAE,OAAO;oCACb,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;oCACxB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;iCAChC,CAAC,CAAC,CAAC,SAAS,CAAC;gCACd,KAAK,GAAG;oCACP,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;oCACrB,IAAI;oCACJ,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;iCACnE,CAAC;4BACH,CAAC;iCAAM,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gCAChC,KAAK,GAAG,IAAI,IAAI,SAAS,CAAC;4BAC3B,CAAC;4BACD,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;wBACnG,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BAChB,MAAM,CAAC,WAAW,CAAC;gCAClB,IAAI,EAAE,aAAa;gCACnB,EAAE,EAAE,OAAO,CAAC,EAAE;gCACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;6BAC7D,CAAC,CAAC;wBACJ,CAAC;gCAAS,CAAC;4BACV,QAAQ,EAAE,CAAC;wBACZ,CAAC;oBACF,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtF,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;oBAC5B,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,IAAI,KAAK,CAAC,sDAAsD,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC9F,CAAC,CAAC,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;gBACjH,KAAK,CAAC,KAAK,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YACpC,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAC1B,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}

package/dist/code-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/code-worker.js

@@ -0,0 +1,120 @@
+import vm from "node:vm";
+import { parentPort, workerData } from "node:worker_threads";
+import { z } from "zod";
+const MAX_BRIDGE_BYTES = 1_000_000;
+const MAX_EMITS = 100;
+const bridgeStringSchema = z.string();
+// SAFETY: ComputerUseCodeExecutor is the sole worker creator and supplies this exact cloneable shape.
+const input = workerData;
+const port = parentPort;
+if (!port)
+    throw new Error("Computer Use code worker requires a parent port");
+let nextCallId = 1;
+const pending = new Map();
+port.on("message", (message) => {
+    const waiter = pending.get(message.id);
+    if (!waiter)
+        return;
+    pending.delete(message.id);
+    if ("error" in message)
+        waiter.reject(new Error(message.error));
+    else
+        waiter.resolve(message.value);
+});
+let emitCount = 0;
+let emittedBytes = 0;
+const checkBridgeValue = (value, label) => {
+    const bytes = Buffer.byteLength(value, "utf8");
+    if (bytes > MAX_BRIDGE_BYTES)
+        throw new Error(`${label} exceeds ${MAX_BRIDGE_BYTES} bytes`);
+    return bytes;
+};
+const callBridge = (method, args) => {
+    checkBridgeValue(args, "Computer Use arguments");
+    const id = nextCallId++;
+    return new Promise((resolve, reject) => {
+        pending.set(id, { resolve, reject });
+        port.postMessage({ type: "call", id, method, args });
+    });
+};
+const emitValue = (type, value) => {
+    if (emitCount >= MAX_EMITS)
+        throw new Error(`Computer Use code exceeded ${MAX_EMITS} emits`);
+    const bytes = checkBridgeValue(value, "Computer Use emitted value");
+    if (emittedBytes + bytes > MAX_BRIDGE_BYTES) {
+        throw new Error(`Computer Use emitted output exceeds ${MAX_BRIDGE_BYTES} bytes`);
+    }
+    emitCount += 1;
+    emittedBytes += bytes;
+    port.postMessage({ type, value });
+};
+const emitBridge = (value) => emitValue("emit", value);
+const emitImageBridge = (value) => emitValue("emit_image", value);
+Object.setPrototypeOf(callBridge, null);
+Object.setPrototypeOf(emitBridge, null);
+Object.setPrototypeOf(emitImageBridge, null);
+const context = vm.createContext({
+    __callBridge: callBridge,
+    __emitBridge: emitBridge,
+    __emitImageBridge: emitImageBridge,
+    __storeJson: JSON.stringify(input.store),
+}, {
+    codeGeneration: { strings: false, wasm: false },
+    name: "computer-use",
+});
+const bootstrap = new vm.Script(`(() => {
+	const callBridge = globalThis.__callBridge;
+	const emitBridge = globalThis.__emitBridge;
+	const emitImageBridge = globalThis.__emitImageBridge;
+	const storeJson = globalThis.__storeJson;
+	delete globalThis.__callBridge;
+	delete globalThis.__emitBridge;
+	delete globalThis.__emitImageBridge;
+	delete globalThis.__storeJson;
+	const call = async (method, args = {}) => JSON.parse(await callBridge(method, JSON.stringify(args)));
+	globalThis.sky = Object.freeze({
+		list_apps: (args) => call("list_apps", args),
+		get_app_state: (args) => call("get_app_state", args),
+		click: (args) => call("click", args),
+		perform_secondary_action: (args) => call("perform_secondary_action", args),
+		set_value: (args) => call("set_value", args),
+		select_text: (args) => call("select_text", args),
+		scroll: (args) => call("scroll", args),
+		drag: (args) => call("drag", args),
+		press_key: (args) => call("press_key", args),
+		type_text: (args) => call("type_text", args),
+	});
+	globalThis.emit = (value) => emitBridge(JSON.stringify(value));
+	globalThis.emitImage = (value) => {
+		if (value === null) throw new Error("get_app_state returned no screenshot");
+		emitImageBridge(JSON.stringify(value));
+	};
+	globalThis.store = JSON.parse(storeJson);
+})()`);
+const serializedStore = () => {
+    const parsed = bridgeStringSchema.safeParse(new vm.Script("JSON.stringify(store)").runInContext(context));
+    if (!parsed.success)
+        throw new Error("Computer Use store must remain a JSON object");
+    checkBridgeValue(parsed.data, "Computer Use store");
+    return parsed.data;
+};
+try {
+    bootstrap.runInContext(context);
+    port.postMessage({ type: "ready" });
+    const script = new vm.Script(`(async () => {\n${input.code}\n})()`, { filename: "computer-use.js" });
+    await script.runInContext(context);
+    port.postMessage({ type: "done", store: serializedStore() });
+}
+catch (cause) {
+    let error = cause instanceof Error ? cause.message : String(cause);
+    let store;
+    try {
+        store = serializedStore();
+    }
+    catch (storeError) {
+        store = JSON.stringify(input.store);
+        error = storeError instanceof Error ? storeError.message : String(storeError);
+    }
+    port.postMessage({ type: "done", store, error });
+}
+//# sourceMappingURL=code-worker.js.map

package/dist/code-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-worker.js","sourceRoot":"","sources":["../src/code-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;AAEtC,sGAAsG;AACtG,MAAM,KAAK,GAAG,UAAyB,CAAC;AACxC,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,IAAI,CAAC,IAAI;IAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;AAE9E,IAAI,UAAU,GAAG,CAAC,CAAC;AACnB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwE,CAAC;AAEhG,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA0B,EAAE,EAAE;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,IAAI,OAAO,IAAI,OAAO;QAAE,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;;QAC3D,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC,CAAC,CAAC;AAEH,IAAI,SAAS,GAAG,CAAC,CAAC;AAClB,IAAI,YAAY,GAAG,CAAC,CAAC;AACrB,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE;IACjE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,gBAAgB;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,gBAAgB,QAAQ,CAAC,CAAC;IAC5F,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AACF,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,IAAY,EAAmB,EAAE;IACpE,gBAAgB,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;IACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AACF,MAAM,SAAS,GAAG,CAAC,IAA2B,EAAE,KAAa,EAAQ,EAAE;IACtE,IAAI,SAAS,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,QAAQ,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;IACpE,IAAI,YAAY,GAAG,KAAK,GAAG,gBAAgB,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,uCAAuC,gBAAgB,QAAQ,CAAC,CAAC;IAClF,CAAC;IACD,SAAS,IAAI,CAAC,CAAC;IACf,YAAY,IAAI,KAAK,CAAC;IACtB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACnC,CAAC,CAAC;AACF,MAAM,UAAU,GAAG,CAAC,KAAa,EAAQ,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACrE,MAAM,eAAe,GAAG,CAAC,KAAa,EAAQ,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;AAChF,MAAM,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACxC,MAAM,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACxC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;AAE7C,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC;IAChC,YAAY,EAAE,UAAU;IACxB,YAAY,EAAE,UAAU;IACxB,iBAAiB,EAAE,eAAe;IAClC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC;CACxC,EAAE;IACF,cAAc,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE;IAC/C,IAAI,EAAE,cAAc;CACpB,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4B3B,CAAC,CAAC;AAEP,MAAM,eAAe,GAAG,GAAW,EAAE;IACpC,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1G,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACrF,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC,IAAI,CAAC;AACpB,CAAC,CAAC;AAEF,IAAI,CAAC;IACJ,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,mBAAmB,KAAK,CAAC,IAAI,QAAQ,EAAE,EAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACrG,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;AAC9D,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IAChB,IAAI,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACJ,KAAK,GAAG,eAAe,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,UAAU,EAAE,CAAC;QACrB,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpC,KAAK,GAAG,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"}

package/integrations/pi/index.ts

@@ -16,21 +16,46 @@ import {
   type DirectBrokerElicitationRequest,
   type DirectBrokerElicitationResponse,
 } from "../../dist/direct-broker.js";
-import { DirectSessionExecutor } from "../../dist/session-executor.js";
-import {
-  TOOL_INPUT_SCHEMAS,
-  TOOL_METADATA,
-  COMPUTER_USE_METHODS,
-  type DirectMethod,
-  type JsonObject,
-} from "../../dist/tools.js";
+import { ComputerUseCodeExecutor } from "../../dist/code-executor.js";
+import { type JsonObject } from "../../dist/tools.js";
 
 const jsonObjectSchema = z.record(z.string(), z.json());
-const toolNames = COMPUTER_USE_METHODS.map((method) => `computer_use_${method}`);
+const codeParameters = {
+  type: "object",
+  additionalProperties: false,
+  required: ["code"],
+  properties: {
+    code: {
+      type: "string",
+      description: "JavaScript body to execute. Use await sky.<method>(args), emit(value), emitImage(screenshot), and store for state shared across calls.",
+    },
+  },
+} as const;
 
-function titleFor(method: DirectMethod): string {
-  return method.split("_").map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`).join(" ");
-}
+const codeDescription = `Run JavaScript that composes OpenAI's official signed macOS Computer Use methods in one call. No nested model is used.
+
+Available globals:
+- sky.list_apps() -> text app inventory
+- sky.get_app_state({ app, disableDiff? }) -> { app, text, screenshot }
+- sky.click({ app, element_index?, x?, y?, mouse_button?, click_count? })
+- sky.perform_secondary_action({ app, element_index, action })
+- sky.set_value({ app, element_index, value })
+- sky.select_text({ app, element_index, text, prefix?, suffix?, selection? })
+- sky.scroll({ app, element_index, direction, pages? })
+- sky.drag({ app, from_x, from_y, to_x, to_y })
+- sky.press_key({ app, key })
+- sky.type_text({ app, text })
+- emit(value) returns text or JSON to Pi
+- emitImage(state.screenshot) returns a screenshot to Pi
+- store is a persistent JSON object shared across calls
+
+get_app_state may return an accessibility-tree diff after the first inspection. Pass disableDiff: true when you need a fresh full tree.
+
+Example:
+const state = await sky.get_app_state({ app: "TextEdit" });
+emit(state.text);
+
+Batch known actions sequentially, then inspect again before deciding the next step.`;
 
 interface PiContentResult {
   content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }>;
@@ -38,34 +63,44 @@ interface PiContentResult {
 }
 
 export async function toPiContent(content: JsonObject[]): Promise<PiContentResult> {
-  const textBlocks = content
-    .filter((block) => block.type !== "image")
-    .map((block) => String(block.text ?? ""));
-  const truncations = textBlocks.map((text) => truncateHead(text, {
-    maxLines: DEFAULT_MAX_LINES,
-    maxBytes: DEFAULT_MAX_BYTES,
-  }));
-  let fullOutputPath: string | undefined;
-  if (truncations.some((result) => result.truncated)) {
-    const tempDir = await mkdtemp(path.join(tmpdir(), "pi-computer-use-"));
-    fullOutputPath = path.join(tempDir, "output.txt");
-    await writeFile(fullOutputPath, textBlocks.join("\n\n"), { encoding: "utf8", mode: 0o600 });
+  const textBlocks = content.filter((block) => block.type !== "image").map((block) => String(block.text ?? ""));
+  const fullText = textBlocks.join("\n\n");
+  const aggregate = truncateHead(fullText, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
+  if (!aggregate.truncated) {
+    return {
+      content: content.map((block) => block.type === "image"
+        ? { type: "image", data: String(block.data), mimeType: String(block.mimeType) }
+        : { type: "text", text: String(block.text ?? "") }),
+    };
   }
 
-  const result: PiContentResult = { content: [] };
+  const tempDir = await mkdtemp(path.join(tmpdir(), "pi-computer-use-"));
+  const fullOutputPath = path.join(tempDir, "output.txt");
+  await writeFile(fullOutputPath, fullText, { encoding: "utf8", mode: 0o600 });
+  const suffix = `\n\n[Official Computer Use text truncated: showing ${aggregate.outputLines} of ${aggregate.totalLines} lines (${formatSize(aggregate.outputBytes)} of ${formatSize(aggregate.totalBytes)}). Full output saved to: ${fullOutputPath}]`;
+  const result: PiContentResult = { content: [], fullOutputPath };
   let textIndex = 0;
+  let textOffset = 0;
+  let noticeAdded = false;
   for (const block of content) {
     if (block.type === "image") {
       result.content.push({ type: "image", data: String(block.data), mimeType: String(block.mimeType) });
       continue;
     }
-    const truncated = truncations[textIndex++];
-    const suffix = truncated.truncated
-      ? `\n\n[Official Computer Use text truncated: showing ${truncated.outputLines} of ${truncated.totalLines} lines (${formatSize(truncated.outputBytes)} of ${formatSize(truncated.totalBytes)}). Full output saved to: ${fullOutputPath}]`
+    const blockText = String(block.text ?? "");
+    const start = textOffset + (textIndex > 0 ? 2 : 0);
+    const end = start + blockText.length;
+    const retained = aggregate.content.length > start
+      ? blockText.slice(0, Math.min(end, aggregate.content.length) - start)
       : "";
-    result.content.push({ type: "text", text: `${truncated.content}${suffix}` });
+    const cutoffHere: boolean = !noticeAdded && aggregate.content.length < end;
+    if (retained || cutoffHere) {
+      result.content.push({ type: "text", text: `${retained}${cutoffHere ? suffix : ""}` });
+      noticeAdded ||= cutoffHere;
+    }
+    textOffset = end;
+    textIndex += 1;
   }
-  if (fullOutputPath) result.fullOutputPath = fullOutputPath;
   return result;
 }
 
@@ -127,7 +162,7 @@ export async function handleOfficialElicitation(
 
 export default function directComputerUse(pi: ExtensionAPI) {
   const stateRoot = process.env.CODEX_COMPUTER_USE_HOME || path.join(getAgentDir(), "direct-computer-use");
-  const sessionExecutor = new DirectSessionExecutor();
+  const codeExecutor = new ComputerUseCodeExecutor();
 
   pi.registerCommand("computer-use-status", {
     description: "Show Computer Use status",
@@ -136,50 +171,45 @@ export default function directComputerUse(pi: ExtensionAPI) {
     },
   });
 
-  for (const method of COMPUTER_USE_METHODS) {
-    const piName = `computer_use_${method}`;
-    pi.registerTool({
-      name: piName,
-      label: titleFor(method),
-      description: TOOL_METADATA[method].description,
-      // SAFETY: Pi accepts standard JSON Schema objects; these schemas are generated directly from the tool's Zod parser.
-      parameters: TOOL_INPUT_SCHEMAS[method] as any,
-      async execute(_toolCallId, params, signal, _onUpdate, ctx) {
-        const parsedParams = jsonObjectSchema.parse(params);
-        const response = await sessionExecutor.execute(method, parsedParams, {
-          stateRoot,
-          signal,
-          supportsOpenAiFormElicitation: true,
-          onElicitation: (request) => handleOfficialElicitation(
-            request,
-            ctx,
-            async (url) => (await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 })).code === 0,
-          ),
-        });
-        if (response.isError) {
-          const message = response.content
-            .filter((block) => block.type === "text")
-            .map((block) => String(block.text ?? ""))
-            .join("\n")
-            .slice(0, 2_000);
-          throw new Error(message || "Official Computer Use returned an error");
-        }
-        const rendered = await toPiContent(response.content);
-        const details: JsonObject = {};
-        if (response.structuredContent !== undefined) details.officialStructuredContent = response.structuredContent;
-        if (rendered.fullOutputPath) details.fullOutputPath = rendered.fullOutputPath;
-        return { content: rendered.content, details };
-      },
-      renderCall(args, theme) {
-        const parsed = z.object({ app: z.string().optional() }).safeParse(args);
-        const app = parsed.success ? parsed.data.app : undefined;
-        const label = theme.fg("toolTitle", theme.bold(piName));
-        return new Text(app ? `${label} ${theme.fg("accent", app)}` : label, 0, 0);
-      },
-    });
-  }
+  pi.registerTool({
+    name: "computer_use",
+    label: "Computer Use",
+    description: codeDescription,
+    promptSnippet: "Run composable JavaScript against OpenAI's official signed macOS Computer Use surface",
+    promptGuidelines: [
+      "Use computer_use for macOS app UI work, composing known sequential actions in one JavaScript call and emitting only the state needed for the next decision.",
+    ],
+    // SAFETY: Pi accepts this standard JSON Schema object as a custom-tool parameter schema.
+    parameters: codeParameters as any,
+    async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+      const { code } = z.object({ code: z.string() }).parse(params);
+      const result = await codeExecutor.execute(code, {
+        stateRoot,
+        signal,
+        supportsOpenAiFormElicitation: true,
+        onElicitation: (request) => handleOfficialElicitation(
+          request,
+          ctx,
+          async (url) => (await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 })).code === 0,
+        ),
+      });
+      const rendered = await toPiContent(result.content);
+      const details: JsonObject = { calls: result.calls };
+      if (result.error) details.error = result.error;
+      if (rendered.fullOutputPath) details.fullOutputPath = rendered.fullOutputPath;
+      return { content: rendered.content, details };
+    },
+    renderCall(args, theme) {
+      const parsed = z.object({ code: z.string() }).safeParse(args);
+      const firstLine = parsed.success
+        ? parsed.data.code.split("\n").map((line) => line.trim()).find(Boolean)?.slice(0, 100)
+        : undefined;
+      const label = theme.fg("toolTitle", theme.bold("computer_use"));
+      return new Text(firstLine ? `${label} ${theme.fg("dim", firstLine)}` : label, 0, 0);
+    },
+  });
 
-  pi.on("session_start", () => pi.setActiveTools([...new Set([...pi.getActiveTools(), ...toolNames])]));
-  pi.on("agent_settled", () => sessionExecutor.close());
-  pi.on("session_shutdown", () => sessionExecutor.close());
+  pi.on("session_start", () => pi.setActiveTools([...new Set([...pi.getActiveTools(), "computer_use"])]));
+  pi.on("agent_settled", () => codeExecutor.close());
+  pi.on("session_shutdown", () => codeExecutor.close());
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-computer-use-mcp",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Expose the official macOS Computer Use tools directly to Pi and MCP clients without a nested model.",
   "author": "Thomas Mustier",
   "license": "MIT",