@oh-my-pi/pi-coding-agent 15.12.3 → 15.13.0

package/src/tools/browser/cmux/rpc.ts

@@ -0,0 +1,156 @@
+import type { Observation, ObservationEntry } from "../tab-protocol";
+
+export interface CmuxKind {
+	kind: "cmux";
+	socketPath: string;
+	password?: string;
+	surface?: string;
+}
+
+export interface CmuxOpenSplitResult {
+	surface_id?: unknown;
+	url?: unknown;
+	workspace_id?: unknown;
+	created_split?: unknown;
+	placement_strategy?: unknown;
+}
+
+export interface CmuxSnapshotRef {
+	role?: unknown;
+	name?: unknown;
+}
+
+export interface CmuxSnapshotPage {
+	title?: unknown;
+	url?: unknown;
+	ready_state?: unknown;
+	text?: unknown;
+	html?: unknown;
+}
+
+export interface CmuxSnapshotResult {
+	snapshot?: unknown;
+	refs?: Record<string, CmuxSnapshotRef>;
+	page?: CmuxSnapshotPage;
+	url?: unknown;
+	title?: unknown;
+	ready_state?: unknown;
+	surface_id?: unknown;
+}
+
+export interface CmuxEvalResult {
+	value?: unknown;
+	surface_id?: unknown;
+	content_world?: unknown;
+}
+
+export interface CmuxUrlGetResult {
+	url?: unknown;
+	surface_id?: unknown;
+	workspace_id?: unknown;
+}
+
+export interface CmuxScreenshotResult {
+	png_base64?: unknown;
+	path?: unknown;
+	url?: unknown;
+	surface_id?: unknown;
+	width?: unknown;
+	height?: unknown;
+}
+
+export interface CmuxGeometry {
+	innerWidth: number;
+	innerHeight: number;
+	dpr: number;
+	scrollX: number;
+	scrollY: number;
+	scrollWidth: number;
+	scrollHeight: number;
+}
+
+export const GEOMETRY_SCRIPT =
+	"(() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, dpr: window.devicePixelRatio||1, scrollX: window.scrollX, scrollY: window.scrollY, scrollWidth: document.documentElement.scrollWidth, scrollHeight: document.documentElement.scrollHeight }))()";
+
+export function cmuxSnapshotToObservation(
+	result: CmuxSnapshotResult,
+	viewport: Observation["viewport"],
+	geometry: CmuxGeometry,
+): Observation {
+	const elements: ObservationEntry[] = [];
+	const refs = result.refs ?? {};
+	for (const ref in refs) {
+		const value = refs[ref];
+		if (!value) continue;
+		const id = Number.parseInt(ref.replace(/^@?e/, ""), 10);
+		if (Number.isNaN(id)) continue;
+		const role = typeof value.role === "string" && value.role.length > 0 ? value.role : "generic";
+		const name = typeof value.name === "string" && value.name.length > 0 ? value.name : undefined;
+		elements.push({ id, role, name, states: [] });
+	}
+	elements.sort((a, b) => a.id - b.id);
+
+	const url =
+		(typeof result.url === "string" && result.url.length > 0 ? result.url : undefined) ??
+		(typeof result.page?.url === "string" && result.page.url.length > 0 ? result.page.url : undefined) ??
+		"about:blank";
+	const title =
+		(typeof result.title === "string" && result.title.length > 0 ? result.title : undefined) ??
+		(typeof result.page?.title === "string" && result.page.title.length > 0 ? result.page.title : undefined);
+	return {
+		url,
+		title,
+		viewport,
+		scroll: {
+			x: geometry.scrollX,
+			y: geometry.scrollY,
+			width: geometry.innerWidth,
+			height: geometry.innerHeight,
+			scrollWidth: geometry.scrollWidth,
+			scrollHeight: geometry.scrollHeight,
+		},
+		elements,
+	};
+}
+
+export function serializeEval(fn: string | ((...args: unknown[]) => unknown), args: unknown[]): string {
+	if (typeof fn === "string") {
+		return fn;
+	}
+	return `(${fn.toString()})(${args.map(arg => JSON.stringify(arg)).join(",")})`;
+}
+
+export function mapWaitUntil(waitUntil: string | undefined): "interactive" | "complete" {
+	return waitUntil === "domcontentloaded" ? "interactive" : "complete";
+}
+
+const TRUTHY_ENV_VALUES = new Set(["1", "Y", "y", "TRUE", "true", "YES", "yes", "ON", "on"]);
+
+function resolveCmuxEnabled(envValue: string | undefined, settingEnabled: boolean): boolean {
+	if (!envValue) return settingEnabled;
+	return TRUTHY_ENV_VALUES.has(envValue);
+}
+
+export interface ResolveCmuxKindOptions {
+	surface?: string;
+	settingEnabled?: boolean;
+}
+
+export function resolveCmuxKind(
+	options?: ResolveCmuxKindOptions | null,
+	env: Record<string, string | undefined> = process.env,
+): CmuxKind | null {
+	if (!resolveCmuxEnabled(env.PI_BROWSER_CMUX, options?.settingEnabled ?? true)) {
+		return null;
+	}
+	const socketPath = env.CMUX_SOCKET_PATH;
+	if (!socketPath) {
+		return null;
+	}
+	return {
+		kind: "cmux",
+		socketPath,
+		password: env.CMUX_SOCKET_PASSWORD || undefined,
+		surface: options?.surface,
+	};
+}

package/src/tools/browser/cmux/socket-client.ts

@@ -0,0 +1,309 @@
+import { randomUUID } from "node:crypto";
+import * as net from "node:net";
+import { ToolError } from "../../tool-errors";
+
+const DEFAULT_CONNECT_TIMEOUT_MS = 10_000;
+const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+
+type RequestJob = {
+	method: string;
+	params: Record<string, unknown>;
+	timeoutMs: number;
+	resolve: (value: Record<string, unknown>) => void;
+	reject: (error: unknown) => void;
+};
+
+type LineWaiter = {
+	resolve: (line: string) => void;
+	reject: (error: unknown) => void;
+	timer: NodeJS.Timeout;
+};
+
+type CmuxErrorPayload = {
+	code?: unknown;
+	message?: unknown;
+	details?: unknown;
+};
+
+export function formatCmuxError(error: CmuxErrorPayload | undefined): string {
+	const code = typeof error?.code === "string" && error.code.length > 0 ? error.code : "error";
+	const message = typeof error?.message === "string" && error.message.length > 0 ? error.message : "cmux error";
+	const details = error?.details === undefined ? "" : ` details=${JSON.stringify(error.details)}`;
+	return `${code}: ${message}${details}`;
+}
+
+export class CmuxSocketClient {
+	readonly #socketPath: string;
+	readonly #password: string | undefined;
+	#socket: net.Socket | null = null;
+	#connectPromise: Promise<void> | null = null;
+	#connected = false;
+	#disposed = false;
+	#buffer = "";
+	readonly #lineWaiters: LineWaiter[] = [];
+	readonly #queue: RequestJob[] = [];
+	#activeJob: RequestJob | null = null;
+	#pumping = false;
+
+	constructor(opts: { socketPath: string; password?: string }) {
+		this.#socketPath = opts.socketPath;
+		this.#password = opts.password;
+	}
+
+	async connect(): Promise<void> {
+		if (this.#disposed) {
+			throw new ToolError("cmux socket closed");
+		}
+		if (this.#connected && this.#socket && !this.#socket.destroyed) {
+			return;
+		}
+		if (this.#connectPromise) {
+			return await this.#connectPromise;
+		}
+
+		this.#connectPromise = this.#openSocket();
+		try {
+			await this.#connectPromise;
+		} finally {
+			this.#connectPromise = null;
+		}
+	}
+
+	async request(
+		method: string,
+		params: Record<string, unknown>,
+		opts?: { timeoutMs?: number },
+	): Promise<Record<string, unknown>> {
+		if (this.#disposed) {
+			throw new ToolError("cmux socket closed");
+		}
+		const { promise, resolve, reject } = Promise.withResolvers<Record<string, unknown>>();
+		this.#queue.push({
+			method,
+			params,
+			timeoutMs: opts?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
+			resolve,
+			reject,
+		});
+		this.#pump();
+		return await promise;
+	}
+
+	close(): void {
+		this.#disposed = true;
+		const err = new ToolError("cmux socket closed");
+		this.#rejectAll(err);
+		this.#socket?.end();
+		this.#socket?.destroy();
+		this.#socket = null;
+		this.#connected = false;
+		this.#connectPromise = null;
+	}
+
+	async #openSocket(): Promise<void> {
+		const socket = net.createConnection({ path: this.#socketPath });
+		this.#socket = socket;
+		this.#buffer = "";
+		socket.setEncoding("utf8");
+		socket.on("data", chunk => this.#onData(String(chunk)));
+		socket.on("error", err => this.#handleSocketFailure(err));
+		socket.on("close", () => this.#handleSocketClose());
+
+		try {
+			await this.#waitForConnect(socket);
+			this.#connected = true;
+			if (this.#password) {
+				const line = await this.#sendLine(`auth ${this.#password}`, DEFAULT_CONNECT_TIMEOUT_MS);
+				if (line.startsWith("ERROR:") && !line.includes("Unknown command 'auth'")) {
+					throw new ToolError(line);
+				}
+			}
+		} catch (err) {
+			this.#connected = false;
+			socket.destroy();
+			if (err instanceof ToolError) throw err;
+			throw new ToolError(
+				`Failed to connect to cmux socket at ${this.#socketPath}: ${err instanceof Error ? err.message : String(err)}`,
+			);
+		}
+	}
+
+	#waitForConnect(socket: net.Socket): Promise<void> {
+		const { promise, resolve, reject } = Promise.withResolvers<void>();
+		const timer = setTimeout(() => {
+			socket.destroy();
+			reject(new ToolError(`Failed to connect to cmux socket at ${this.#socketPath}: timed out`));
+		}, DEFAULT_CONNECT_TIMEOUT_MS);
+		const cleanup = (): void => {
+			clearTimeout(timer);
+			socket.off("connect", onConnect);
+			socket.off("error", onError);
+		};
+		const onConnect = (): void => {
+			cleanup();
+			resolve();
+		};
+		const onError = (err: Error): void => {
+			cleanup();
+			reject(new ToolError(`Failed to connect to cmux socket at ${this.#socketPath}: ${err.message}`));
+		};
+		socket.once("connect", onConnect);
+		socket.once("error", onError);
+		return promise;
+	}
+
+	#pump(): void {
+		if (this.#pumping) return;
+		this.#pumping = true;
+		void this.#pumpLoop();
+	}
+
+	async #pumpLoop(): Promise<void> {
+		try {
+			while (this.#queue.length > 0 && !this.#disposed) {
+				const job = this.#queue.shift();
+				if (!job) continue;
+				this.#activeJob = job;
+				try {
+					await this.connect();
+					const request = JSON.stringify({ id: randomUUID(), method: job.method, params: job.params });
+					const line = await this.#sendLine(request, job.timeoutMs);
+					job.resolve(this.#parseResponse(line));
+				} catch (err) {
+					job.reject(err instanceof Error ? err : new ToolError(String(err)));
+				} finally {
+					if (this.#activeJob === job) {
+						this.#activeJob = null;
+					}
+				}
+			}
+		} finally {
+			this.#pumping = false;
+			if (this.#queue.length > 0 && !this.#disposed) {
+				this.#pump();
+			}
+		}
+	}
+
+	#sendLine(line: string, timeoutMs: number): Promise<string> {
+		if (!this.#socket || this.#socket.destroyed) {
+			throw new ToolError("cmux socket is not connected");
+		}
+		const read = this.#nextLine(timeoutMs);
+		this.#socket.write(`${line}\n`, err => {
+			if (err) {
+				this.#handleSocketFailure(err);
+			}
+		});
+		return read;
+	}
+
+	#nextLine(timeoutMs: number): Promise<string> {
+		const { promise, resolve, reject } = Promise.withResolvers<string>();
+		let waiter: LineWaiter;
+		const timer = setTimeout(() => {
+			const index = this.#lineWaiters.indexOf(waiter);
+			if (index >= 0) {
+				this.#lineWaiters.splice(index, 1);
+			}
+			reject(new ToolError("Timed out waiting for cmux socket response"));
+			this.#destroySocketForDesync();
+		}, timeoutMs);
+		waiter = {
+			resolve: line => {
+				clearTimeout(timer);
+				resolve(line);
+			},
+			reject: err => {
+				clearTimeout(timer);
+				reject(err);
+			},
+			timer,
+		};
+		this.#lineWaiters.push(waiter);
+		this.#drainLines();
+		return promise;
+	}
+
+	#onData(chunk: string): void {
+		this.#buffer += chunk;
+		this.#drainLines();
+	}
+
+	#drainLines(): void {
+		while (this.#lineWaiters.length > 0) {
+			const newlineIndex = this.#buffer.indexOf("\n");
+			if (newlineIndex < 0) return;
+			let line = this.#buffer.slice(0, newlineIndex);
+			this.#buffer = this.#buffer.slice(newlineIndex + 1);
+			if (line.endsWith("\r")) {
+				line = line.slice(0, -1);
+			}
+			const waiter = this.#lineWaiters.shift();
+			waiter?.resolve(line);
+		}
+	}
+
+	#parseResponse(line: string): Record<string, unknown> {
+		if (line.startsWith("ERROR:")) {
+			throw new ToolError(line);
+		}
+		let payload: unknown;
+		try {
+			payload = JSON.parse(line);
+		} catch (err) {
+			throw new ToolError(`Invalid cmux socket JSON response: ${err instanceof Error ? err.message : String(err)}`);
+		}
+		if (!payload || typeof payload !== "object") {
+			throw new ToolError("Invalid cmux socket response: expected object");
+		}
+		const response = payload as { ok?: unknown; result?: unknown; error?: CmuxErrorPayload };
+		if (response.ok === true) {
+			return (response.result ?? {}) as Record<string, unknown>;
+		}
+		if (response.ok === false) {
+			throw new ToolError(formatCmuxError(response.error));
+		}
+		throw new ToolError("Invalid cmux socket response: missing ok flag");
+	}
+
+	#handleSocketFailure(err: Error): void {
+		if (this.#disposed) return;
+		this.#connected = false;
+		this.#connectPromise = null;
+		this.#rejectAll(new ToolError(`cmux socket error: ${err.message}`));
+		this.#socket?.destroy();
+		this.#socket = null;
+	}
+
+	#handleSocketClose(): void {
+		const hadPendingRead = this.#lineWaiters.length > 0;
+		this.#connected = false;
+		this.#connectPromise = null;
+		this.#socket = null;
+		if (!this.#disposed && hadPendingRead) {
+			this.#rejectAll(new ToolError("cmux socket closed"));
+		}
+	}
+
+	#destroySocketForDesync(): void {
+		this.#connected = false;
+		this.#connectPromise = null;
+		this.#socket?.destroy();
+		this.#socket = null;
+	}
+
+	#rejectAll(err: ToolError): void {
+		for (const waiter of this.#lineWaiters.splice(0)) {
+			clearTimeout(waiter.timer);
+			waiter.reject(err);
+		}
+		if (this.#activeJob) {
+			this.#activeJob.reject(err);
+			this.#activeJob = null;
+		}
+		for (const job of this.#queue.splice(0)) {
+			job.reject(err);
+		}
+	}
+}

package/src/tools/browser/registry.ts

@@ -4,26 +4,42 @@ import type { Subprocess } from "bun";
 import type { Browser, CDPSession } from "puppeteer-core";
 import { ToolAbortError, ToolError } from "../tool-errors";
 import { findFreeCdpPort, findReusableCdp, gracefulKillTreeOnce, killExistingByPath, waitForCdp } from "./attach";
+import type { CmuxKind } from "./cmux/rpc";
+import { CmuxSocketClient } from "./cmux/socket-client";
 import { BROWSER_PROTOCOL_TIMEOUT_MS, launchHeadlessBrowser, loadPuppeteer, type UserAgentOverride } from "./launch";
 
-export type BrowserKind =
+export type PuppeteerBrowserKind =
 	| { kind: "headless"; headless: boolean }
 	| { kind: "spawned"; path: string }
 	| { kind: "connected"; cdpUrl: string };
 
+export type BrowserKind = PuppeteerBrowserKind | CmuxKind;
+
 export type BrowserKindTag = BrowserKind["kind"];
 
-export interface BrowserHandle {
+interface BrowserHandleCommon {
 	key: string;
 	kind: BrowserKind;
+	refCount: number;
+}
+
+export interface PuppeteerBrowserHandle extends BrowserHandleCommon {
+	kind: PuppeteerBrowserKind;
 	browser: Browser;
 	cdpUrl?: string;
 	pid?: number;
 	subprocess?: Subprocess;
-	refCount: number;
 	stealth: { browserSession: CDPSession | null; override: UserAgentOverride | null };
 }
 
+export interface CmuxBrowserHandle extends BrowserHandleCommon {
+	kind: CmuxKind;
+	client: CmuxSocketClient;
+	surface?: string;
+}
+
+export type BrowserHandle = PuppeteerBrowserHandle | CmuxBrowserHandle;
+
 const browsers = new Map<string, BrowserHandle>();
 
 function browserKey(kind: BrowserKind): string {
@@ -34,6 +50,8 @@ function browserKey(kind: BrowserKind): string {
 			return `spawned:${kind.path}`;
 		case "connected":
 			return `connected:${kind.cdpUrl}`;
+		case "cmux":
+			return `cmux:${kind.socketPath}`;
 	}
 }
 
@@ -48,6 +66,7 @@ export async function acquireBrowser(kind: BrowserKind, opts: AcquireBrowserOpti
 	const key = browserKey(kind);
 	const existing = browsers.get(key);
 	if (existing) {
+		if ("client" in existing) return existing;
 		if (existing.browser.connected) return existing;
 		browsers.delete(key);
 		await disposeBrowserHandle(existing, { kill: false });
@@ -69,6 +88,17 @@ export function normalizeConnectedCdpUrl(rawCdpUrl: string): string {
 }
 
 async function openBrowserHandle(kind: BrowserKind, opts: AcquireBrowserOptions): Promise<BrowserHandle> {
+	if (kind.kind === "cmux") {
+		const client = new CmuxSocketClient({ socketPath: kind.socketPath, password: kind.password });
+		await client.connect();
+		return {
+			key: browserKey(kind),
+			kind,
+			client,
+			surface: kind.surface,
+			refCount: 0,
+		};
+	}
 	if (kind.kind === "headless") {
 		const browser = await launchHeadlessBrowser({ headless: kind.headless, viewport: opts.viewport });
 		return {
@@ -176,6 +206,10 @@ export async function releaseBrowser(handle: BrowserHandle, opts: { kill: boolea
 }
 
 async function disposeBrowserHandle(handle: BrowserHandle, opts: { kill: boolean }): Promise<void> {
+	if ("client" in handle) {
+		handle.client.close();
+		return;
+	}
 	if (handle.kind.kind === "headless") {
 		if (handle.browser.connected) {
 			try {

package/src/tools/browser/render.ts

@@ -23,7 +23,7 @@ interface BrowserRenderArgs {
 	code?: string;
 	all?: boolean;
 	kill?: boolean;
-	app?: { path?: string; cdp_url?: string; target?: string };
+	app?: { path?: string; cdp_url?: string; target?: string; cmux?: boolean; surface?: string };
 	viewport?: { width: number; height: number; scale?: number };
 	timeout?: number;
 }
@@ -36,6 +36,9 @@ interface BrowserRenderContext {
 function describeBrowser(args: BrowserRenderArgs, details: BrowserToolDetails | undefined): string | undefined {
 	if (args.app?.cdp_url) return `connected ${args.app.cdp_url}`;
 	if (args.app?.path) return `spawned ${shortenPath(args.app.path)}`;
+	if (args.app?.cmux !== false && (args.app?.cmux === true || args.app?.surface)) {
+		return args.app.surface ? `cmux ${args.app.surface}` : "cmux";
+	}
 	switch (details?.browser) {
 		case "headless":
 			return "headless";
@@ -43,6 +46,8 @@ function describeBrowser(args: BrowserRenderArgs, details: BrowserToolDetails |
 			return "spawned";
 		case "connected":
 			return "connected";
+		case "cmux":
+			return "cmux";
 		default:
 			return undefined;
 	}

package/src/tools/browser/tab-protocol.ts

@@ -38,6 +38,8 @@ export interface ScreenshotResult {
 export interface SessionSnapshot {
 	cwd: string;
 	browserScreenshotDir?: string;
+	/** Force non-WebP screenshot encoding (e.g. for Ollama). Unset honors `OMP_NO_WEBP`. */
+	excludeWebP?: boolean;
 }
 
 export type WorkerInitPayload =