@oh-my-pi/pi-coding-agent 17.0.9 → 17.1.1

package/src/tools/browser.ts

@@ -7,9 +7,24 @@ import type { ToolSession } from "../sdk";
 import { enforceInlineByteCap } from "../session/streaming-output";
 import { truncateForPrompt } from "./approval";
 import { resolveCmuxKind } from "./browser/cmux/rpc";
-import { acquireBrowser, type BrowserHandle, type BrowserKind, type BrowserKindTag } from "./browser/registry";
+import {
+	acquireBrowser,
+	type BrowserHandle,
+	type BrowserKind,
+	type BrowserKindTag,
+	holdBrowser,
+	releaseBrowser,
+} from "./browser/registry";
 import type { Observation, ScreenshotResult } from "./browser/tab-protocol";
-import { acquireTab, dropHeadlessTabs, getTab, releaseAllTabs, releaseTab, runInTab } from "./browser/tab-supervisor";
+import {
+	type AcquireTabResult,
+	acquireTab,
+	dropHeadlessTabs,
+	getTab,
+	releaseAllTabs,
+	releaseTab,
+	runInTab,
+} from "./browser/tab-supervisor";
 import type { OutputMeta } from "./output-meta";
 import { resolveToCwd } from "./path-utils";
 import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors";
@@ -232,52 +247,84 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
 			);
 		}
 
-		const browser = await untilAborted(signal, () =>
-			acquireBrowser(kind, {
-				cwd: this.session.cwd,
-				viewport: params.viewport
-					? {
-							width: params.viewport.width,
-							height: params.viewport.height,
-							deviceScaleFactor: params.viewport.scale,
-						}
-					: undefined,
-				appArgs: params.app?.args,
-				signal,
-			}),
-		);
+		// The requested timeout must cover the *entire* open — browser
+		// acquisition (CDP discovery/connect), queued tab acquisition, worker
+		// creation, and navigation — not only `acquireTab`. Compose one deadline
+		// from the caller signal and `params.timeout` and thread it through both
+		// stages so a stalled acquisition rejects at the requested boundary.
+		const timeoutSignal = AbortSignal.timeout(timeoutMs);
+		const openSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+		try {
+			const browser = await untilAborted(openSignal, () =>
+				acquireBrowser(kind, {
+					cwd: this.session.cwd,
+					viewport: params.viewport
+						? {
+								width: params.viewport.width,
+								height: params.viewport.height,
+								deviceScaleFactor: params.viewport.scale,
+							}
+						: undefined,
+					appArgs: params.app?.args,
+					signal: openSignal,
+				}),
+			);
+
+			// Hold one open-acquisition lease across the whole tab acquisition.
+			// A freshly-created browser sits in the registry at refCount 0 until a
+			// tab takes a hold; without this lease an abort/timeout mid-acquisition
+			// (or a sibling open of a different tab name on the same browser that
+			// fails) could dispose it out from under this operation. The lease is
+			// released exactly once — the success and failure paths are mutually
+			// exclusive — transferring ownership to the published tab on success or
+			// rolling the fresh browser back on failure.
+			holdBrowser(browser);
+			let result: AcquireTabResult;
+			try {
+				result = await untilAborted(openSignal, () =>
+					acquireTab(name, browser, {
+						url: params.url,
+						waitUntil: params.wait_until,
+						viewport: params.viewport
+							? {
+									width: params.viewport.width,
+									height: params.viewport.height,
+									deviceScaleFactor: params.viewport.scale,
+								}
+							: undefined,
+						target: params.app?.target,
+						timeoutMs,
+						dialogs: params.dialogs,
+						signal: openSignal,
+						ownerSessionId: this.session.getSessionId?.() ?? undefined,
+					}),
+				);
+			} catch (error) {
+				await releaseBrowser(browser, { kill: false });
+				throw error;
+			}
+			await releaseBrowser(browser, { kill: false });
 
-		const result = await untilAborted(signal, () =>
-			acquireTab(name, browser, {
-				url: params.url,
-				waitUntil: params.wait_until,
-				viewport: params.viewport
-					? {
-							width: params.viewport.width,
-							height: params.viewport.height,
-							deviceScaleFactor: params.viewport.scale,
-						}
-					: undefined,
-				target: params.app?.target,
-				timeoutMs,
-				dialogs: params.dialogs,
-				signal,
-				ownerSessionId: this.session.getSessionId?.() ?? undefined,
-			}),
-		);
-		const tab = result.tab;
-		const url = tab.info.url;
-		const title = tab.info.title ?? "";
-		details.url = url;
-		details.viewport = tab.info.viewport;
-		const verb = result.created ? "Opened" : "Reused";
-		const lines = [
-			`${verb} tab ${JSON.stringify(name)} on ${describeBrowser(browser)}`,
-			`URL: ${url}`,
-			title ? `Title: ${title}` : null,
-		].filter((l): l is string => typeof l === "string");
-		details.result = lines.join("\n");
-		return toolResult(details).text(lines.join("\n")).done();
+			const tab = result.tab;
+			const url = tab.info.url;
+			const title = tab.info.title ?? "";
+			details.url = url;
+			details.viewport = tab.info.viewport;
+			const verb = result.created ? "Opened" : "Reused";
+			const lines = [
+				`${verb} tab ${JSON.stringify(name)} on ${describeBrowser(browser)}`,
+				`URL: ${url}`,
+				title ? `Title: ${title}` : null,
+			].filter((l): l is string => typeof l === "string");
+			details.result = lines.join("\n");
+			return toolResult(details).text(lines.join("\n")).done();
+		} catch (error) {
+			// Caller cancellation stays a ToolAbortError; the requested timeout
+			// becomes a timeout ToolError; anything else passes through unchanged.
+			if (signal?.aborted) throw error instanceof ToolAbortError ? error : new ToolAbortError();
+			if (timeoutSignal.aborted) throw new ToolError(`Browser open timed out after ${timeoutMs}ms`);
+			throw error;
+		}
 	}
 
 	async #close(

package/src/tools/builtin-names.ts

@@ -13,6 +13,7 @@ export const BUILTIN_TOOL_NAMES = [
 	"lsp",
 	"inspect_image",
 	"browser",
+	"computer",
 	"checkpoint",
 	"rewind",
 	"task",

package/src/tools/computer/protocol.ts

@@ -0,0 +1,28 @@
+import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
+
+export const COMPUTER_WORKER_ARG = "__omp_worker_computer";
+
+export type ComputerWorkerInbound =
+	| { type: "ping"; id: string }
+	| { type: "init"; options: DesktopSessionOptions }
+	| { type: "execute"; id: string; actions: DesktopAction[] }
+	| { type: "close" };
+
+export type ComputerWorkerOutbound =
+	| { type: "pong"; id: string }
+	| { type: "ready"; capabilities: DesktopCapabilities }
+	| { type: "result"; id: string; capture: DesktopCapture; capabilities: DesktopCapabilities }
+	| { type: "error"; id?: string; error: ComputerWorkerError }
+	| { type: "closed" };
+
+export interface ComputerWorkerError {
+	name: string;
+	message: string;
+	stack?: string;
+}
+
+export interface ComputerWorkerTransport {
+	send(message: ComputerWorkerOutbound, transfer?: Bun.Transferable[]): void;
+	onMessage(handler: (message: ComputerWorkerInbound) => void): () => void;
+	close(): void;
+}

package/src/tools/computer/supervisor.ts

@@ -0,0 +1,258 @@
+import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
+import { withTimeout, workerHostEntry } from "@oh-my-pi/pi-utils";
+import { ToolAbortError, ToolError } from "../tool-errors";
+import {
+	COMPUTER_WORKER_ARG,
+	type ComputerWorkerError,
+	type ComputerWorkerInbound,
+	type ComputerWorkerOutbound,
+} from "./protocol";
+
+const START_TIMEOUT_MS = 10_000;
+const CLOSE_TIMEOUT_MS = 1_500;
+const SMOKE_TIMEOUT_MS = 5_000;
+
+export interface ComputerController {
+	readonly capabilities: DesktopCapabilities | undefined;
+	execute(actions: DesktopAction[], signal?: AbortSignal): Promise<DesktopCapture>;
+	close(): Promise<void>;
+}
+
+export interface ComputerWorkerHandle {
+	send(message: ComputerWorkerInbound): void;
+	onMessage(handler: (message: ComputerWorkerOutbound) => void): () => void;
+	onError(handler: (error: Error) => void): () => void;
+	terminate(): Promise<void>;
+}
+
+export interface ComputerSupervisorTimeouts {
+	startMs: number;
+	closeMs: number;
+}
+
+const DEFAULT_TIMEOUTS: ComputerSupervisorTimeouts = {
+	startMs: START_TIMEOUT_MS,
+	closeMs: CLOSE_TIMEOUT_MS,
+};
+export type ComputerWorkerFactory = () => ComputerWorkerHandle;
+
+function workerError(error: ComputerWorkerError): Error {
+	const result = new ToolError(error.message);
+	result.name = error.name;
+	if (error.stack) result.stack = error.stack;
+	return result;
+}
+
+function wrapWorker(worker: Worker): ComputerWorkerHandle {
+	return {
+		send(message) {
+			worker.postMessage(message);
+		},
+		onMessage(handler) {
+			const listener = (event: MessageEvent): void => handler(event.data as ComputerWorkerOutbound);
+			worker.addEventListener("message", listener);
+			return () => worker.removeEventListener("message", listener);
+		},
+		onError(handler) {
+			const listener = (event: ErrorEvent): void =>
+				handler(event.error instanceof Error ? event.error : new Error(event.message));
+			worker.addEventListener("error", listener);
+			return () => worker.removeEventListener("error", listener);
+		},
+		async terminate() {
+			worker.terminate();
+		},
+	};
+}
+
+export function spawnComputerWorker(): ComputerWorkerHandle {
+	const hostEntry = workerHostEntry();
+	const worker = hostEntry
+		? new Worker(hostEntry, { type: "module", argv: [COMPUTER_WORKER_ARG] })
+		: new Worker(new URL("./worker-entry.ts", import.meta.url).href, { type: "module" });
+	return wrapWorker(worker);
+}
+
+interface PendingRequest {
+	resolve(capture: DesktopCapture): void;
+	reject(error: unknown): void;
+}
+
+export class ComputerSupervisor implements ComputerController {
+	#worker?: ComputerWorkerHandle;
+	#startPromise?: Promise<void>;
+	#capabilities?: DesktopCapabilities;
+	#pending = new Map<string, PendingRequest>();
+	#nextId = 0;
+	#serial: Promise<void> = Promise.resolve();
+	#closed = false;
+	#unsubscribeMessage?: () => void;
+	#unsubscribeError?: () => void;
+
+	constructor(
+		private readonly options: DesktopSessionOptions,
+		private readonly createWorker: ComputerWorkerFactory = spawnComputerWorker,
+		private readonly timeouts: ComputerSupervisorTimeouts = DEFAULT_TIMEOUTS,
+	) {}
+
+	get capabilities(): DesktopCapabilities | undefined {
+		return this.#capabilities;
+	}
+
+	execute(actions: DesktopAction[], signal?: AbortSignal): Promise<DesktopCapture> {
+		if (this.#closed) return Promise.reject(new ToolError("Computer session is closed"));
+		const run = (): Promise<DesktopCapture> => this.#execute(actions, signal);
+		const result = this.#serial.then(run, run);
+		this.#serial = result.then(
+			() => undefined,
+			() => undefined,
+		);
+		return result;
+	}
+
+	async #execute(actions: DesktopAction[], signal?: AbortSignal): Promise<DesktopCapture> {
+		if (this.#closed) throw new ToolError("Computer session is closed");
+		if (signal?.aborted) throw new ToolAbortError();
+		await this.#start();
+		if (signal?.aborted) throw new ToolAbortError();
+		const id = `computer-${++this.#nextId}`;
+		const request = Promise.withResolvers<DesktopCapture>();
+		this.#pending.set(id, request);
+		this.#worker!.send({ type: "execute", id, actions });
+		if (!signal) return request.promise;
+
+		const aborted = Promise.withResolvers<never>();
+		const onAbort = (): void => aborted.reject(new ToolAbortError());
+		signal.addEventListener("abort", onAbort, { once: true });
+		try {
+			return await Promise.race([request.promise, aborted.promise]);
+		} catch (error) {
+			if (error instanceof ToolAbortError) await this.#terminate(error);
+			throw error;
+		} finally {
+			signal.removeEventListener("abort", onAbort);
+		}
+	}
+
+	#start(): Promise<void> {
+		if (this.#startPromise) return this.#startPromise;
+		const ready = Promise.withResolvers<void>();
+		try {
+			const worker = this.createWorker();
+			this.#worker = worker;
+			this.#unsubscribeMessage = worker.onMessage(message => {
+				if (message.type === "ready") {
+					this.#capabilities = message.capabilities;
+					ready.resolve();
+					return;
+				}
+				if (message.type === "result") {
+					this.#capabilities = message.capabilities;
+					const pending = this.#pending.get(message.id);
+					this.#pending.delete(message.id);
+					pending?.resolve(message.capture);
+					return;
+				}
+				if (message.type === "error") {
+					const error = workerError(message.error);
+					if (message.id) {
+						const pending = this.#pending.get(message.id);
+						this.#pending.delete(message.id);
+						pending?.reject(error);
+					} else {
+						ready.reject(error);
+					}
+				}
+			});
+			this.#unsubscribeError = worker.onError(error => {
+				ready.reject(error);
+				void this.#terminate(error);
+			});
+			worker.send({ type: "init", options: this.options });
+		} catch (error) {
+			ready.reject(error);
+		}
+		this.#startPromise = withTimeout(
+			ready.promise,
+			this.timeouts.startMs,
+			"Timed out starting native computer worker",
+		).catch(async error => {
+			await this.#terminate(error);
+			throw error;
+		});
+		return this.#startPromise;
+	}
+
+	async #terminate(reason: unknown): Promise<void> {
+		const worker = this.#worker;
+		this.#worker = undefined;
+		this.#startPromise = undefined;
+		this.#capabilities = undefined;
+		this.#unsubscribeMessage?.();
+		this.#unsubscribeMessage = undefined;
+		this.#unsubscribeError?.();
+		this.#unsubscribeError = undefined;
+		for (const pending of this.#pending.values()) pending.reject(reason);
+		this.#pending.clear();
+		await worker?.terminate();
+	}
+
+	async close(): Promise<void> {
+		if (this.#closed) return;
+		this.#closed = true;
+		const worker = this.#worker;
+		if (!worker) return;
+		const closed = Promise.withResolvers<void>();
+		const unsubscribe = worker.onMessage(message => {
+			if (message.type === "closed") closed.resolve();
+		});
+		try {
+			worker.send({ type: "close" });
+			await withTimeout(closed.promise, this.timeouts.closeMs, "Timed out closing native computer worker");
+		} catch {
+			// Forced termination below is the bounded close fallback.
+		} finally {
+			unsubscribe();
+			await this.#terminate(new ToolError("Computer session closed"));
+		}
+	}
+}
+
+const ownedSupervisors = new Map<string, Set<ComputerController>>();
+
+export function registerComputerController(ownerId: string | undefined, controller: ComputerController): () => void {
+	if (!ownerId) return () => {};
+	const controllers = ownedSupervisors.get(ownerId) ?? new Set<ComputerController>();
+	controllers.add(controller);
+	ownedSupervisors.set(ownerId, controllers);
+	return () => {
+		controllers.delete(controller);
+		if (controllers.size === 0) ownedSupervisors.delete(ownerId);
+	};
+}
+
+export async function releaseComputerSessionsForOwner(ownerId: string | undefined): Promise<void> {
+	if (!ownerId) return;
+	const controllers = ownedSupervisors.get(ownerId);
+	if (!controllers) return;
+	ownedSupervisors.delete(ownerId);
+	await Promise.allSettled(Array.from(controllers, controller => controller.close()));
+}
+
+export async function smokeTestComputerWorker(timeoutMs = SMOKE_TIMEOUT_MS): Promise<void> {
+	const worker = spawnComputerWorker();
+	const id = "computer-smoke";
+	const pong = Promise.withResolvers<void>();
+	const unsubscribeMessage = worker.onMessage(message => {
+		if (message.type === "pong" && message.id === id) pong.resolve();
+	});
+	const unsubscribeError = worker.onError(error => pong.reject(error));
+	try {
+		worker.send({ type: "ping", id });
+		await withTimeout(pong.promise, timeoutMs, "Computer worker smoke ping timed out");
+	} finally {
+		unsubscribeMessage();
+		unsubscribeError();
+		await worker.terminate();
+	}
+}

package/src/tools/computer/worker-entry.ts

@@ -0,0 +1,25 @@
+import { parentPort } from "node:worker_threads";
+import { consumeWorkerInbox } from "@oh-my-pi/pi-utils/worker-host";
+import type { ComputerWorkerInbound, ComputerWorkerTransport } from "./protocol";
+import { ComputerWorkerCore } from "./worker";
+
+if (!parentPort) throw new Error("computer-worker-entry: missing parentPort");
+
+const port = parentPort;
+const inbox = consumeWorkerInbox();
+const transport: ComputerWorkerTransport = {
+	send(message, transfer) {
+		port.postMessage(message, transfer ?? []);
+	},
+	onMessage(handler) {
+		if (inbox) return inbox.bind(message => handler(message as ComputerWorkerInbound));
+		const listener = (message: unknown): void => handler(message as ComputerWorkerInbound);
+		port.on("message", listener);
+		return () => port.off("message", listener);
+	},
+	close() {
+		port.close();
+	},
+};
+
+new ComputerWorkerCore(transport);

package/src/tools/computer/worker.ts

@@ -0,0 +1,135 @@
+import {
+	type DesktopAction,
+	type DesktopCapture,
+	DesktopSession,
+	type DesktopSessionOptions,
+} from "@oh-my-pi/pi-natives";
+import type { ComputerWorkerError, ComputerWorkerInbound, ComputerWorkerTransport } from "./protocol";
+
+export interface NativeDesktopSession {
+	readonly capabilities: DesktopSession["capabilities"];
+	capture(): Promise<DesktopCapture>;
+	execute(actions: DesktopAction[]): Promise<DesktopCapture>;
+	close(): Promise<void>;
+}
+
+export type NativeDesktopSessionFactory = (options: DesktopSessionOptions) => NativeDesktopSession;
+
+const COORDINATE_ACTIONS: ReadonlySet<DesktopAction["type"]> = new Set([
+	"click",
+	"double_click",
+	"drag",
+	"move",
+	"scroll",
+]);
+
+function serializeError(error: unknown): ComputerWorkerError {
+	if (error instanceof Error) {
+		return { name: error.name, message: error.message, ...(error.stack ? { stack: error.stack } : {}) };
+	}
+	return { name: "Error", message: String(error) };
+}
+
+function captureTransfer(capture: DesktopCapture): Bun.Transferable[] {
+	const buffer = capture.data.buffer;
+	return buffer instanceof ArrayBuffer ? [buffer] : [];
+}
+
+export class ComputerWorkerCore {
+	#session?: NativeDesktopSession;
+	#hasReturnedFrame = false;
+	#closed = false;
+	#tail: Promise<void> = Promise.resolve();
+	readonly #unsubscribe: () => void;
+
+	constructor(
+		private readonly transport: ComputerWorkerTransport,
+		private readonly createSession: NativeDesktopSessionFactory = options => new DesktopSession(options),
+	) {
+		this.#unsubscribe = transport.onMessage(message => this.#onMessage(message));
+	}
+
+	#onMessage(message: ComputerWorkerInbound): void {
+		if (message.type === "ping") {
+			this.transport.send({ type: "pong", id: message.id });
+			return;
+		}
+		if (message.type === "close") {
+			this.#tail = this.#tail.then(() => this.#close());
+			return;
+		}
+		if (message.type === "init") {
+			this.#tail = this.#tail.then(() => this.#init(message.options));
+			return;
+		}
+		this.#tail = this.#tail.then(() => this.#execute(message.id, message.actions));
+	}
+
+	async #init(options: DesktopSessionOptions): Promise<void> {
+		if (this.#closed) return;
+		if (this.#session) {
+			this.transport.send({
+				type: "error",
+				error: { name: "Error", message: "Computer worker already initialized" },
+			});
+			return;
+		}
+		try {
+			this.#session = this.createSession(options);
+			this.#hasReturnedFrame = false;
+			this.transport.send({ type: "ready", capabilities: this.#session.capabilities });
+		} catch (error) {
+			this.transport.send({ type: "error", error: serializeError(error) });
+		}
+	}
+
+	async #execute(id: string, actions: DesktopAction[]): Promise<void> {
+		const session = this.#session;
+		if (!session) {
+			this.transport.send({
+				type: "error",
+				id,
+				error: { name: "Error", message: "Computer worker is not initialized" },
+			});
+			return;
+		}
+		try {
+			if (!this.#hasReturnedFrame && actions.some(action => COORDINATE_ACTIONS.has(action.type))) {
+				this.transport.send({
+					type: "error",
+					id,
+					error: {
+						name: "Error",
+						message:
+							"Coordinate computer actions require a screenshot returned to the provider; request a screenshot first",
+					},
+				});
+				return;
+			}
+			const capture = await session.execute(actions);
+			this.transport.send(
+				{ type: "result", id, capture, capabilities: session.capabilities },
+				captureTransfer(capture),
+			);
+			this.#hasReturnedFrame = true;
+		} catch (error) {
+			this.transport.send({ type: "error", id, error: serializeError(error) });
+		}
+	}
+
+	async #close(): Promise<void> {
+		if (this.#closed) return;
+		this.#closed = true;
+		try {
+			await this.#session?.close();
+		} catch (error) {
+			this.transport.send({ type: "error", error: serializeError(error) });
+		} finally {
+			this.#session = undefined;
+			this.#hasReturnedFrame = false;
+			this.#unsubscribe();
+			this.transport.send({ type: "closed" });
+			this.transport.close();
+		}
+	}
+}