@oh-my-pi/pi-coding-agent 17.2.4 → 17.2.5

package/src/tools/browser/render.ts

@@ -24,7 +24,7 @@ interface BrowserRenderArgs {
 	code?: string;
 	all?: boolean;
 	kill?: boolean;
-	app?: { path?: string; cdp_url?: string; target?: string; cmux?: boolean; surface?: string };
+	app?: { path?: string; cdp_url?: string; relay?: boolean; target?: string; cmux?: boolean; surface?: string };
 	viewport?: { width: number; height: number; scale?: number };
 	timeout?: number;
 }
@@ -39,6 +39,7 @@ function describeBrowser(args: BrowserRenderArgs, details: BrowserToolDetails |
 	if (cdpUrl) return `connected ${cdpUrl}`;
 	const appPath = typeof args.app?.path === "string" ? args.app.path : "";
 	if (appPath) return `spawned ${shortenPath(appPath)}`;
+	if (args.app?.relay) return "relay";
 	if (args.app?.cmux !== false && (args.app?.cmux === true || args.app?.surface)) {
 		return args.app.surface ? `cmux ${args.app.surface}` : "cmux";
 	}
@@ -49,6 +50,8 @@ function describeBrowser(args: BrowserRenderArgs, details: BrowserToolDetails |
 			return "spawned";
 		case "connected":
 			return "connected";
+		case "relay":
+			return "relay";
 		case "cmux":
 			return "cmux";
 		default:

package/src/tools/browser/shared-daemon.ts

@@ -11,7 +11,8 @@
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { logger } from "@oh-my-pi/pi-utils";
-import { type DaemonBrokerClient, daemonClientForProject } from "../../launch/client";
+import { daemonClientForProject } from "../../launch/client";
+import { describeQuietly, stopQuietly, waitReady } from "../../launch/ensure";
 import { daemonRuntimeDir } from "../../launch/paths";
 import type { DaemonSnapshot } from "../../launch/protocol";
 import { throwIfAborted } from "../tool-errors";
@@ -20,7 +21,6 @@ import { resolveSharedBrowserLaunchSpec } from "./launch";
 /** Chrome prints this on stderr once the CDP listener is up; the broker's ready probe captures the line. */
 const READY_LOG_PATTERN = String.raw`DevTools listening on ws://\S+`;
 const READY_TIMEOUT_MS = 30_000;
-const STOP_TIMEOUT_MS = 5_000;
 const PROBE_TIMEOUT_MS = 1_500;
 /** describe→start rounds before giving up; bounds cross-process start races and wedged-Chrome replacement. */
 const ENSURE_ATTEMPTS = 3;
@@ -59,58 +59,6 @@ async function probeEndpoint(wsEndpoint: string): Promise<boolean> {
 	}
 }
 
-/** Snapshot the daemon, treating "unknown daemon" as absent. */
-async function describeQuietly(
-	client: DaemonBrokerClient,
-	name: string,
-	signal?: AbortSignal,
-): Promise<DaemonSnapshot | undefined> {
-	try {
-		const result = await client.request({ op: "describe", name }, signal);
-		return result.op === "describe" ? result.daemon : undefined;
-	} catch (error) {
-		throwIfAborted(signal);
-		logger.debug("Shared browser describe failed", {
-			name,
-			error: error instanceof Error ? error.message : String(error),
-		});
-		return undefined;
-	}
-}
-
-/** Block until the daemon reports ready; undefined on timeout or pre-ready exit. */
-async function waitReady(
-	client: DaemonBrokerClient,
-	name: string,
-	signal?: AbortSignal,
-): Promise<DaemonSnapshot | undefined> {
-	try {
-		const result = await client.request({ op: "wait", name, for: "ready", timeoutMs: READY_TIMEOUT_MS }, signal);
-		if (result.op !== "wait" || result.timedOut) return undefined;
-		return result.daemon;
-	} catch (error) {
-		throwIfAborted(signal);
-		logger.debug("Shared browser ready wait failed", {
-			name,
-			error: error instanceof Error ? error.message : String(error),
-		});
-		return undefined;
-	}
-}
-
-/** Best-effort stop before replacing a wedged or endpoint-less daemon. */
-async function stopQuietly(client: DaemonBrokerClient, name: string, signal?: AbortSignal): Promise<void> {
-	try {
-		await client.request({ op: "stop", name, timeoutMs: STOP_TIMEOUT_MS }, signal);
-	} catch (error) {
-		throwIfAborted(signal);
-		logger.debug("Shared browser stop failed", {
-			name,
-			error: error instanceof Error ? error.message : String(error),
-		});
-	}
-}
-
 /**
  * Ensure the project-shared automation Chromium is running and reachable,
  * launching it under the daemon broker when needed. Idempotent across
@@ -139,16 +87,17 @@ export async function ensureSharedBrowser(opts: {
 	await fs.mkdir(userDataDir, { recursive: true });
 	for (let attempt = 0; attempt < ENSURE_ATTEMPTS; attempt++) {
 		throwIfAborted(opts.signal);
-		const existing = await describeQuietly(client, name, opts.signal);
+		const existing = await describeQuietly(client, name, "Shared browser", opts.signal);
 		if (existing && existing.state !== "exited" && existing.state !== "failed") {
-			const settled = existing.readyAt !== undefined ? existing : await waitReady(client, name, opts.signal);
+			const settled =
+				existing.readyAt !== undefined ? existing : await waitReady(client, name, "Shared browser", opts.signal);
 			const wsEndpoint = wsEndpointOf(settled);
 			if (wsEndpoint && (await probeEndpoint(wsEndpoint))) {
 				return { wsEndpoint, daemonName: name, projectDir: client.projectDir };
 			}
 			// Live record but unreachable Chrome (wedged, or readiness never
 			// matched): replace it rather than handing out a dead endpoint.
-			await stopQuietly(client, name, opts.signal);
+			await stopQuietly(client, name, "Shared browser", opts.signal);
 			continue;
 		}
 		try {
@@ -175,7 +124,7 @@ export async function ensureSharedBrowser(opts: {
 			if (wsEndpoint && (await probeEndpoint(wsEndpoint))) {
 				return { wsEndpoint, daemonName: name, projectDir: client.projectDir };
 			}
-			await stopQuietly(client, name, opts.signal);
+			await stopQuietly(client, name, "Shared browser", opts.signal);
 		} catch (error) {
 			throwIfAborted(opts.signal);
 			// Lost a cross-process start race ("already starting/ready"); the next

package/src/tools/browser/tab-supervisor.ts

@@ -698,10 +698,11 @@ async function buildInitPayload(browser: PuppeteerBrowserHandle, opts: AcquireTa
 			timeoutMs: opts.timeoutMs,
 		};
 	}
-	// A connected browser is user-driven. When no target is requested, adopt its
-	// visible tab and avoid raising it before screenshots. An explicit target may
-	// be backgrounded, so retain activation to guarantee target-correct pixels.
-	const activateForScreenshot = browser.kind.kind !== "connected" || !shouldPreserveConnectedBrowserFocus(opts.target);
+	// Connected and relay browsers are user-driven. When no target is requested,
+	// adopt the visible tab and avoid raising it before screenshots. An explicit
+	// target may be backgrounded, so retain activation for target-correct pixels.
+	const userDriven = browser.kind.kind === "connected" || browser.kind.kind === "relay";
+	const activateForScreenshot = !userDriven || !shouldPreserveConnectedBrowserFocus(opts.target);
 	const page = await pickElectronTarget(browser.browser, {
 		matcher: opts.target,
 		preferVisible: !activateForScreenshot,

package/src/tools/browser/tab-worker.ts

@@ -20,6 +20,14 @@ import { JsRuntime, type RuntimeHooks } from "../../eval/js/shared/runtime";
 import { resizeImage } from "../../utils/image-resize";
 import { resolveToCwd } from "../path-utils";
 import { formatScreenshot } from "../render-utils";
+import {
+	bindRunFacade,
+	CELL_BUDGET_SLACK_MS,
+	markHandled,
+	resolvePredicateTimeout,
+	type WaitPredicateOptions,
+	waitForRun,
+} from "../run-scope";
 import { ToolAbortError, ToolError, throwIfAborted } from "../tool-errors";
 import {
 	type AriaSnapshotOptions,
@@ -36,14 +44,6 @@ import {
 	loadPuppeteerInWorker,
 } from "./launch";
 import { extractReadableFromHtml, type ReadableFormat } from "./readable";
-import {
-	bindBrowserRunFacade,
-	CELL_BUDGET_SLACK_MS,
-	markHandled,
-	resolvePredicateTimeout,
-	type WaitPredicateOptions,
-	waitForBrowserRun,
-} from "./run-cancellation";
 import { cloneSafe, RunOutput } from "./run-output";
 import type {
 	Observation,
@@ -827,6 +827,7 @@ export class WorkerCore {
 				const page = await target.page();
 				if (!page) throw new ToolError(`Target ${payload.targetId} is no longer available on the attached browser`);
 				this.#page = page;
+				await this.#claimRelayTarget(page);
 				this.#observeDialogs();
 				if (payload.dialogs) this.#applyDialogPolicy(payload.dialogs);
 				if (payload.url) {
@@ -853,6 +854,26 @@ export class WorkerCore {
 		throw new ToolError(`Target ${targetId} is no longer available on the attached browser`);
 	}
 
+	/**
+	 * Tell the omp browser relay this worker drives the adopted page, so the
+	 * relay adds it to the per-window "omp" tab group. Best-effort: plain CDP
+	 * backends (real Chrome, cmux) reject the relay-private method.
+	 */
+	async #claimRelayTarget(page: Page): Promise<void> {
+		let session: CDPSession | undefined;
+		try {
+			session = await page.createCDPSession();
+			// Puppeteer's protocol map cannot express the relay-private method; the
+			// send signature is otherwise identical.
+			const raw = session as unknown as { send(method: string): Promise<unknown> };
+			await raw.send("OMP.claimTarget");
+		} catch {
+			// Not the omp relay; nothing to claim.
+		} finally {
+			await session?.detach().catch(() => undefined);
+		}
+	}
+
 	/**
 	 * Best-effort unblocking of a wedged target during post-timeout recovery: dismiss any
 	 * open JS dialog and stop a pending navigation over a raw CDP session (created on the
@@ -975,9 +996,9 @@ export class WorkerCore {
 			const runtime = this.#ensureRuntime(msg.session);
 			runtime.setCwd(msg.session.cwd);
 			runtime.setRunScope({
-				page: bindBrowserRunFacade(runPage.page, signal),
-				browser: bindBrowserRunFacade(browser, signal),
-				tab: bindBrowserRunFacade(tabApi, signal),
+				page: bindRunFacade(runPage.page, signal),
+				browser: bindRunFacade(browser, signal),
+				tab: bindRunFacade(tabApi, signal),
 				assert: (cond: unknown, text?: string): void => {
 					if (!cond) throw new ToolError(text ?? "Assertion failed");
 				},
@@ -991,7 +1012,7 @@ export class WorkerCore {
 							: { timeout: resolvePredicateTimeout(msg.timeoutMs, opts?.timeout), interval: opts?.interval };
 					return markHandled(
 						this.#runOp(active, label, signal, Number.POSITIVE_INFINITY, sig =>
-							waitForBrowserRun(msOrPredicate, sig, resolved),
+							waitForRun(msOrPredicate, sig, resolved),
 						),
 					);
 				},

package/src/tools/browser.ts

@@ -15,6 +15,7 @@ import {
 	holdBrowser,
 	releaseBrowser,
 } from "./browser/registry";
+import { resolveRelayKind } from "./browser/relay/kind";
 import type { Observation, ScreenshotResult } from "./browser/tab-protocol";
 import {
 	type AcquireTabResult,
@@ -39,6 +40,7 @@ export {
 export { cmuxSnapshotToObservation, mapWaitUntil, resolveCmuxKind, serializeEval } from "./browser/cmux/rpc";
 export { CmuxSocketClient } from "./browser/cmux/socket-client";
 export { extractReadableFromHtml, type ReadableFormat, type ReadableResult } from "./browser/readable";
+export { DEFAULT_RELAY_URL, type RelayKind, resolveRelayKind } from "./browser/relay/kind";
 export type { Observation, ObservationEntry } from "./browser/tab-protocol";
 
 const DEFAULT_TAB_NAME = "main";
@@ -46,6 +48,7 @@ const DEFAULT_TAB_NAME = "main";
 const appSchema = type({
 	"path?": type("string").describe("binary path to spawn"),
 	"cdp_url?": type("string").describe("existing cdp endpoint"),
+	"relay?": type("boolean").describe("drive the user's own tabs via the omp browser relay"),
 	"args?": type("string[]").describe("extra cli args"),
 	"target?": type("string").describe("substring to pick a window"),
 });
@@ -95,7 +98,24 @@ function resolveBrowserKind(params: BrowserParams, session: ToolSession): Browse
 		const exe = resolveToCwd(app.path, session.cwd);
 		return { kind: "spawned", path: exe };
 	}
-	// A configured endpoint is a default, not an override: explicit app options win.
+	const relayUrl = session.settings.get("browser.relayUrl") as string | undefined;
+	// Explicit app.relay wins over every setting; PI_BROWSER_RELAY stays the
+	// final kill switch (a relay that is down would otherwise brick the tool).
+	if (app?.relay) {
+		const relayKind = resolveRelayKind({ settingEnabled: true, url: relayUrl });
+		if (relayKind) return relayKind;
+	}
+	// Relay before cdpUrl among settings: enabling the opt-out-by-default relay
+	// is a deliberate mode selection, while cdpUrl is a standing fallback
+	// endpoint. A configured endpoint is a default, not an override: explicit
+	// app options win.
+	if (app?.relay !== false) {
+		const relayKind = resolveRelayKind({
+			settingEnabled: session.settings.get("browser.relay") as boolean | undefined,
+			url: relayUrl,
+		});
+		if (relayKind) return relayKind;
+	}
 	const configuredCdpUrl = (session.settings.get("browser.cdpUrl") as string | undefined)?.trim();
 	if (configuredCdpUrl) {
 		return { kind: "connected", cdpUrl: configuredCdpUrl.replace(/\/+$/, "") };
@@ -427,6 +447,8 @@ function describeBrowser(handle: BrowserHandle): string {
 			return `spawned ${handle.kind.path} (pid ${handle.pid ?? "?"})`;
 		case "connected":
 			return `connected ${handle.cdpUrl ?? handle.kind.cdpUrl}`;
+		case "relay":
+			return `relay ${handle.cdpUrl ?? handle.kind.cdpUrl}`;
 	}
 }
 
@@ -438,6 +460,8 @@ function describeKind(kind: BrowserKind): string {
 			return `spawned:${kind.path}`;
 		case "connected":
 			return `connected:${kind.cdpUrl}`;
+		case "relay":
+			return `relay:${kind.cdpUrl}`;
 		case "cmux":
 			return `cmux:${kind.surface ?? "split"}`;
 	}
@@ -448,6 +472,7 @@ function sameBrowserKind(a: BrowserKind, b: BrowserKind): boolean {
 	if (a.kind === "headless" && b.kind === "headless") return a.headless === b.headless;
 	if (a.kind === "spawned" && b.kind === "spawned") return a.path === b.path;
 	if (a.kind === "connected" && b.kind === "connected") return a.cdpUrl === b.cdpUrl;
+	if (a.kind === "relay" && b.kind === "relay") return a.cdpUrl === b.cdpUrl;
 	if (a.kind === "cmux" && b.kind === "cmux") return a.socketPath === b.socketPath;
 	return false;
 }

package/src/tools/computer/exposure.ts

@@ -1,38 +1,14 @@
 import type { Model } from "@oh-my-pi/pi-ai";
 
-export type ComputerExposureMode = "native" | "function" | "unavailable";
-
-export interface ComputerExposureOptions {
-	azureBaseUrl?: string;
-	azureResourceName?: string;
-}
-
-function isFirstPartyAzureEndpoint(baseUrl: string): boolean {
-	try {
-		const url = new URL(baseUrl);
-		return (
-			url.protocol === "https:" &&
-			(url.hostname.endsWith(".openai.azure.com") || url.hostname === "models.inference.ai.azure.com")
-		);
-	} catch {
-		return false;
-	}
-}
-
-/** Match the provider transport's effective Computer Use tool representation. */
-export function computerExposureMode(
-	model: Model | undefined,
-	options?: ComputerExposureOptions,
-): ComputerExposureMode {
-	if (!model) return "unavailable";
-	if (model.supportsComputerUse !== true) return "function";
-	if (model.api !== "azure-openai-responses" || model.supportsComputerUseConfig !== undefined) return "native";
-	const baseUrl =
-		options?.azureBaseUrl?.trim() ||
-		process.env.AZURE_OPENAI_BASE_URL?.trim() ||
-		(options?.azureResourceName || process.env.AZURE_OPENAI_RESOURCE_NAME
-			? `https://${options?.azureResourceName || process.env.AZURE_OPENAI_RESOURCE_NAME}.openai.azure.com/openai/v1`
-			: undefined) ||
-		model.baseUrl;
-	return baseUrl && isFirstPartyAzureEndpoint(baseUrl) ? "native" : "function";
+/** How the computer tool is represented to the active model. */
+export type ComputerExposureMode = "function" | "unavailable";
+
+/**
+ * Report the computer tool's callable representation.
+ *
+ * Window selection is part of the tool's JSON schema, so even models with
+ * provider-native Computer Use support receive it as a function.
+ */
+export function computerExposureMode(model: Model | undefined): ComputerExposureMode {
+	return model ? "function" : "unavailable";
 }

package/src/tools/computer/protocol.ts

@@ -1,26 +1,67 @@
-import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
+import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
+import type { DesktopCapabilities } from "@oh-my-pi/pi-natives";
 
+/** Hidden CLI selector that re-enters the computer worker host. */
 export const COMPUTER_WORKER_ARG = "__omp_worker_computer";
 
+/** Frozen run settings transferred from the tool session to the worker. */
+export interface ComputerSessionSnapshot {
+	cwd: string;
+	sessionId: string;
+	captureMaxWidth: number;
+	captureMaxHeight: number;
+	display: string;
+	readOnly: boolean;
+}
+
+/** Reply envelope for a session tool invoked by desktop JavaScript. */
+export type ToolReply = { ok: true; value: unknown } | { ok: false; error: RunErrorPayload };
+
+/** Commands accepted by the persistent computer worker. */
 export type ComputerWorkerInbound =
 	| { type: "ping"; id: string }
-	| { type: "init"; options: DesktopSessionOptions }
-	| { type: "execute"; id: string; actions: DesktopAction[] }
+	| { type: "run"; id: string; code: string; timeoutMs: number; session: ComputerSessionSnapshot }
+	| { type: "abort"; id: string }
+	| { type: "tool-reply"; id: string; reply: ToolReply }
 	| { type: "close" };
 
-export type ComputerWorkerOutbound =
-	| { type: "pong"; id: string }
-	| { type: "ready"; capabilities: DesktopCapabilities }
-	| { type: "result"; id: string; capture: DesktopCapture; capabilities: DesktopCapabilities }
-	| { type: "error"; id?: string; error: ComputerWorkerError }
-	| { type: "closed" };
+/** Successful computer run output returned to the tool supervisor. */
+export interface ComputerRunOk {
+	displays: Array<TextContent | ImageContent>;
+	returnValue: unknown;
+	screenshots: ComputerScreenshot[];
+	capabilities?: DesktopCapabilities;
+}
 
-export interface ComputerWorkerError {
+/** Full-resolution screenshot emitted during one computer run. */
+export interface ComputerScreenshot {
+	path: string;
+	width: number;
+	height: number;
+	sourceWidth?: number;
+	sourceHeight?: number;
+	target: string;
+}
+
+/** Clone-safe error metadata returned across the worker boundary. */
+export interface RunErrorPayload {
 	name: string;
 	message: string;
 	stack?: string;
+	isToolError: boolean;
+	isAbort: boolean;
 }
 
+/** Events emitted by the persistent computer worker. */
+export type ComputerWorkerOutbound =
+	| { type: "ready" }
+	| { type: "pong"; id: string }
+	| { type: "result"; id: string; ok: true; payload: ComputerRunOk }
+	| { type: "result"; id: string; ok: false; error: RunErrorPayload }
+	| { type: "tool-call"; id: string; runId: string; name: string; args: unknown }
+	| { type: "closed" };
+
+/** Transport used by the worker core in Bun workers and tests. */
 export interface ComputerWorkerTransport {
 	send(message: ComputerWorkerOutbound, transfer?: Bun.Transferable[]): void;
 	onMessage(handler: (message: ComputerWorkerInbound) => void): () => void;