@forsion/tangu-computer-use 0.5.8

package/src/vendor/platform/macos/helper.ts

@@ -0,0 +1,291 @@
+import { spawn } from "node:child_process";
+import { constants as fsConstants } from "node:fs";
+import { access, mkdir, realpath } from "node:fs/promises";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { toBoolean, toFiniteNumber, toOptionalString } from "../coerce.ts";
+import type { PlatformDiagnostics } from "../types.ts";
+import { resolveMacosHelperAppPath } from "./helper-path.mjs";
+
+const COMMAND_TIMEOUT_MS = 15_000;
+const HELPER_PROTOCOL_VERSION = 12;
+const HELPER_SETUP_TIMEOUT_MS = 60_000;
+
+export const HELPER_BUNDLE_ID = "com.forsion.tangu-computer-use";
+export const HELPER_APP_PATH = resolveMacosHelperAppPath();
+export const HELPER_APP_EXECUTABLE_PATH = path.join(HELPER_APP_PATH, "Contents", "MacOS", "bridge");
+const DEFAULT_HELPER_SOCKET_PATH = path.join(os.homedir(), "Library", "Caches", "tangu-computer-use", "bridge.sock");
+export const HELPER_SOCKET_PATH = process.env.PI_CU_SOCKET_PATH ?? DEFAULT_HELPER_SOCKET_PATH;
+const usingExternalHelperSocket = HELPER_SOCKET_PATH !== DEFAULT_HELPER_SOCKET_PATH;
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
+const SETUP_HELPER_SCRIPT = path.join(PACKAGE_ROOT, "scripts", "setup-helper.mjs");
+
+export class HelperTransportError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = "HelperTransportError";
+	}
+}
+
+export class HelperCommandError extends Error {
+	readonly code?: string;
+
+	constructor(message: string, code?: string) {
+		super(message);
+		this.name = "HelperCommandError";
+		this.code = code;
+	}
+}
+
+function throwIfAborted(signal?: AbortSignal): void {
+	if (signal?.aborted) throw new Error("Operation aborted.");
+}
+
+async function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+	throwIfAborted(signal);
+	await new Promise<void>((resolve, reject) => {
+		const timer = setTimeout(resolve, ms);
+		const onAbort = () => {
+			clearTimeout(timer);
+			reject(new Error("Operation aborted."));
+		};
+		signal?.addEventListener("abort", onAbort, { once: true });
+	}).finally(() => signal?.throwIfAborted?.());
+}
+
+async function isExecutable(filePath: string): Promise<boolean> {
+	try {
+		await access(filePath, fsConstants.X_OK);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+async function isResolvedHelperExecutable(filePath?: string): Promise<boolean> {
+	if (!filePath) return true;
+	const [actualPath, expectedPath] = await Promise.all([
+		realpath(filePath).catch(() => path.resolve(filePath)),
+		realpath(HELPER_APP_EXECUTABLE_PATH).catch(() => path.resolve(HELPER_APP_EXECUTABLE_PATH)),
+	]);
+	return actualPath === expectedPath;
+}
+
+export async function runProcess(
+	command: string,
+	args: string[],
+	timeoutMs: number,
+	signal?: AbortSignal,
+	env?: NodeJS.ProcessEnv,
+): Promise<void> {
+	throwIfAborted(signal);
+
+	await new Promise<void>((resolve, reject) => {
+		const child = spawn(command, args, {
+			stdio: ["ignore", "pipe", "pipe"],
+			env,
+		});
+
+		let stderr = "";
+		let stdout = "";
+
+		const timer = setTimeout(() => {
+			child.kill("SIGTERM");
+			cleanup();
+			reject(new Error(`Command timed out after ${timeoutMs}ms: ${command} ${args.join(" ")}`));
+		}, timeoutMs);
+
+		const onAbort = () => {
+			child.kill("SIGTERM");
+			cleanup();
+			reject(new Error("Operation aborted."));
+		};
+
+		const cleanup = () => {
+			clearTimeout(timer);
+			signal?.removeEventListener("abort", onAbort);
+		};
+
+		child.stdout.on("data", (chunk) => {
+			stdout += String(chunk);
+		});
+
+		child.stderr.on("data", (chunk) => {
+			stderr += String(chunk);
+		});
+
+		child.on("error", (error) => {
+			cleanup();
+			reject(error);
+		});
+
+		child.on("close", (code) => {
+			cleanup();
+			if (code === 0) {
+				resolve();
+				return;
+			}
+			const output = [stderr.trim(), stdout.trim()].filter(Boolean).join("\n");
+			reject(new Error(`Command failed (${code}): ${command} ${args.join(" ")}\n${output}`.trim()));
+		});
+
+		signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}
+
+export class MacosHelperClient {
+	private daemonAvailable = false;
+	private requestSequence = 0;
+	private diagnosticsCache?: PlatformDiagnostics;
+
+	get diagnostics(): PlatformDiagnostics | undefined {
+		return this.diagnosticsCache;
+	}
+
+	async ensureInstalled(signal?: AbortSignal): Promise<void> {
+		if (usingExternalHelperSocket) return;
+		// Installation is a deployment/repair operation, not part of every new
+		// agent process's hot path. Protocol compatibility is checked against the
+		// live daemon immediately afterwards.
+		if (await isExecutable(HELPER_APP_EXECUTABLE_PATH)) {
+			return;
+		}
+
+		// Re-enter Electron and Bun standalone hosts as their JavaScript runtimes.
+		await runProcess(process.execPath, [SETUP_HELPER_SCRIPT, "--runtime"], HELPER_SETUP_TIMEOUT_MS, signal, {
+			...process.env,
+			ELECTRON_RUN_AS_NODE: "1",
+			BUN_BE_BUN: "1",
+		});
+
+		if (!(await isExecutable(HELPER_APP_EXECUTABLE_PATH))) {
+			throw new Error(`Failed to install tangu-computer-use helper app at ${HELPER_APP_PATH}.`);
+		}
+	}
+
+	async launchDaemon(signal?: AbortSignal): Promise<void> {
+		if (usingExternalHelperSocket) throw new HelperTransportError(`External helper socket is unavailable at ${HELPER_SOCKET_PATH}.`);
+		await mkdir(path.dirname(HELPER_SOCKET_PATH), { recursive: true });
+		// Open the resolved bundle directly so a legacy system-wide copy with the
+		// same bundle id cannot win LaunchServices resolution.
+		await runProcess("open", ["-n", "-g", HELPER_APP_PATH, "--args", "serve", "--socket", HELPER_SOCKET_PATH], COMMAND_TIMEOUT_MS, signal);
+	}
+
+	async daemonCommand<T>(cmd: string, args: Record<string, unknown>, timeoutMs: number, signal?: AbortSignal): Promise<T> {
+		return await new Promise<T>((resolve, reject) => {
+			const id = `req_${++this.requestSequence}`;
+			const socket = net.createConnection(HELPER_SOCKET_PATH);
+			let buffer = "";
+			const timer = setTimeout(() => { socket.destroy(); reject(new HelperTransportError(`Daemon command '${cmd}' timed out after ${timeoutMs}ms.`)); }, timeoutMs);
+			const cleanup = () => { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); };
+			const onAbort = () => { socket.destroy(); cleanup(); reject(new Error("Operation aborted.")); };
+			signal?.addEventListener("abort", onAbort, { once: true });
+			socket.setEncoding("utf8");
+			socket.on("connect", () => socket.write(`${JSON.stringify({ id, cmd, ...args })}\n`));
+			socket.on("data", (chunk) => {
+				buffer += chunk;
+				const newline = buffer.indexOf("\n");
+				if (newline < 0) return;
+				cleanup();
+				socket.end();
+				try {
+					const parsed = JSON.parse(buffer.slice(0, newline));
+					if (parsed.ok === true) resolve(parsed.result as T);
+					else reject(new HelperCommandError(parsed?.error?.message ?? `Daemon command '${cmd}' failed.`, parsed?.error?.code));
+				} catch (error) {
+					reject(error);
+				}
+			});
+			socket.on("error", (error) => { cleanup(); reject(new HelperTransportError(error.message)); });
+		});
+	}
+
+	async ensureDaemon(signal?: AbortSignal): Promise<boolean> {
+		if (this.daemonAvailable) return true;
+		try {
+			await this.daemonCommand("diagnostics", {}, 1_000, signal);
+			this.daemonAvailable = true;
+			return true;
+		} catch {}
+		await this.launchDaemon(signal).catch(() => undefined);
+		for (let index = 0; index < 30; index += 1) {
+			try {
+				await this.daemonCommand("diagnostics", {}, 1_000, signal);
+				this.daemonAvailable = true;
+				return true;
+			} catch {
+				await sleep(100, signal);
+			}
+		}
+		return false;
+	}
+
+	async command<T>(cmd: string, args: Record<string, unknown> = {}, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<T> {
+		const timeoutMs = options?.timeoutMs ?? COMMAND_TIMEOUT_MS;
+		if (!(await this.ensureDaemon(options?.signal))) {
+			throw new HelperTransportError(`tangu-computer-use helper app daemon is unavailable at ${HELPER_APP_PATH}.`);
+		}
+		try {
+			return await this.daemonCommand<T>(cmd, args, timeoutMs, options?.signal);
+		} catch (error) {
+			this.daemonAvailable = false;
+			throw error instanceof Error ? error : new Error(String(error));
+		}
+	}
+
+	async restart(signal?: AbortSignal): Promise<void> {
+		await this.command("shutdown", {}, { signal, timeoutMs: 2_000 }).catch(() => undefined);
+		this.daemonAvailable = false;
+		await sleep(400, signal);
+		if (!(await this.ensureDaemon(signal))) {
+			throw new Error(`tangu-computer-use helper did not come back after restart. Helper app: ${HELPER_APP_PATH}`);
+		}
+	}
+
+	async diagnosticsCommand(signal?: AbortSignal): Promise<PlatformDiagnostics> {
+		const result = await this.command<any>("diagnostics", {}, { signal });
+		const diagnostics = {
+			protocolVersion: Math.trunc(toFiniteNumber(result?.protocolVersion, 0)),
+			architectureVersion: Math.trunc(toFiniteNumber(result?.architectureVersion, 0)),
+			invariants: Array.isArray(result?.invariants) ? result.invariants.filter((value: unknown): value is string => typeof value === "string") : [],
+			pid: Math.trunc(toFiniteNumber(result?.pid, 0)),
+			parentPid: Math.trunc(toFiniteNumber(result?.parentPid, 0)) || undefined,
+			parentAppName: toOptionalString(result?.parentAppName),
+			parentBundleId: toOptionalString(result?.parentBundleId),
+			parentPath: toOptionalString(result?.parentPath),
+			executablePath: toOptionalString(result?.executablePath),
+			os: toOptionalString(result?.macOS),
+			arch: toOptionalString(result?.arch),
+			accessibility: toBoolean(result?.accessibility),
+			screenRecording: toBoolean(result?.screenRecording),
+		};
+		this.diagnosticsCache = diagnostics;
+		return diagnostics;
+	}
+
+	async ensureProtocol(signal?: AbortSignal): Promise<PlatformDiagnostics> {
+		let diagnostics = await this.diagnosticsCommand(signal);
+		const executableMatches = await isResolvedHelperExecutable(diagnostics.executablePath);
+		if (diagnostics.protocolVersion === HELPER_PROTOCOL_VERSION && executableMatches) return diagnostics;
+
+		// The helper daemon outlives Pi, so restarting/reloading Pi alone does not
+		// replace a stale daemon or one launched from the legacy system location.
+		// Stop it through the backwards-compatible command channel and relaunch
+		// the exact app bundle that ensureInstalled() resolved.
+		await this.restart(signal);
+		diagnostics = await this.diagnosticsCommand(signal);
+		const relaunchedExecutableMatches = await isResolvedHelperExecutable(diagnostics.executablePath);
+		if (diagnostics.protocolVersion !== HELPER_PROTOCOL_VERSION || !relaunchedExecutableMatches) {
+			this.daemonAvailable = false;
+			throw new Error(
+				`tangu-computer-use helper mismatch after relaunch: expected protocol ${HELPER_PROTOCOL_VERSION} and executable ${HELPER_APP_EXECUTABLE_PATH}; got protocol ${diagnostics.protocolVersion} and executable ${diagnostics.executablePath ?? "unknown"}. Reinstall or rebuild the helper app at ${HELPER_APP_PATH}.`,
+			);
+		}
+		return diagnostics;
+	}
+}
+
+export const macosHelper = new MacosHelperClient();

package/src/vendor/platform/macos/permissions.ts

@@ -0,0 +1,146 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { ensurePermissions, type PermissionKind, type PermissionStatus } from "../../permissions.ts";
+import { toBoolean, toFiniteNumber, toOptionalString } from "../coerce.ts";
+import type { PlatformReadyState } from "../types.ts";
+import { HELPER_APP_PATH, macosHelper } from "./helper.ts";
+import { assertPlatformArchitecture } from "../architecture.ts";
+
+const GRANT_INSTRUCTIONS =
+	"Grant Accessibility and Screen Recording to tangu-computer-use.app in System Settings → Privacy & Security. " +
+	"Screen Recording lets the agent see the window; Accessibility lets it interact with the window.";
+
+const SIGNING_MIGRATION_WARNING =
+	"If these permissions were enabled before this install/update, macOS invalidated the old grants because " +
+	"tangu-computer-use.app was re-signed. Re-enable both toggles for the newly signed helper. " +
+	"If a toggle is already on, switch it off and on again.";
+
+const macosPermissionKinds = [
+	{ kind: "accessibility" as const, openOption: "Open Accessibility Settings (missing)" },
+	{ kind: "screenRecording" as const, openOption: "Open Screen Recording Settings (missing)" },
+];
+
+function permissionStatusSummary(status: PermissionStatus): string {
+	const lines = [
+		`Accessibility: ${status.accessibility ? "granted" : "missing"}`,
+		`Screen Recording: ${status.screenRecording ? "granted" : "missing"}`,
+	];
+	if (status.screenRecordingPreflight && !status.screenRecording) {
+		lines.push(
+			"(Screen Recording reads granted in the TCC database but a live capture probe failed — " +
+			"the grant likely belongs to a different app identity, or the helper needs a restart.)",
+		);
+	}
+	return lines.join("; ");
+}
+
+function permissionPrompt(status: PermissionStatus, helperPath: string, hint?: string): string {
+	return [
+		"tangu-computer-use needs macOS permissions for its helper app.",
+		permissionStatusSummary(status),
+		"",
+		`Helper: tangu-computer-use.app (${helperPath})`,
+		hint,
+		"",
+		`Important: ${SIGNING_MIGRATION_WARNING}`,
+		"",
+		"tangu-computer-use.app is already listed in the pane(s) — enable its toggle, then choose Recheck.",
+	].filter(Boolean).join("\n");
+}
+
+function missingPermissionMessage(kinds: PermissionKind[]): string {
+	return `Still missing after restart: ${kinds.join(" and ")}. ${SIGNING_MIGRATION_WARNING} Then choose Recheck again.`;
+}
+
+async function checkPermissions(signal?: AbortSignal): Promise<PermissionStatus> {
+	const result = await macosHelper.command<any>("checkPermissions", {}, { signal });
+	const rawSource = result?.source;
+	return {
+		accessibility: toBoolean(result?.accessibility),
+		// Authoritative: the helper's live ScreenCaptureKit probe.
+		screenRecording: toBoolean(result?.screenRecordingCapturable),
+		// Keep the preflight value separate: disagreement means stale per-process
+		// TCC cache or a grant row belonging to another app identity.
+		screenRecordingPreflight: toBoolean(result?.screenRecordingPreflight),
+		source: rawSource && typeof rawSource === "object"
+			? {
+				// macOS attributes Accessibility / Screen Recording grants to the
+				// responsible process at the top of the launch chain. "helper-app"
+				// is the canonical installed app via LaunchServices; "caller" means
+				// grants would attach to the launching app instead.
+				attribution: rawSource.attribution === "helper-app" ? "helper-app" : "caller",
+				pid: Math.trunc(toFiniteNumber(rawSource.pid, 0)) || undefined,
+				parentPid: Math.trunc(toFiniteNumber(rawSource.parentPid, 0)) || undefined,
+				executablePath: toOptionalString(rawSource.executablePath),
+				parentPath: toOptionalString(rawSource.parentPath),
+				parentBundleId: toOptionalString(rawSource.parentBundleId),
+				os: toOptionalString(rawSource.macOS),
+			}
+			: undefined,
+	};
+}
+
+async function registerPermissions(signal?: AbortSignal): Promise<void> {
+	// Raises the Accessibility prompt and performs a real ScreenCaptureKit
+	// capture attempt so tangu-computer-use.app is pre-listed in both Settings
+	// panes; the user only flips toggles, no "+" path picking.
+	await macosHelper.command("registerPermissions", {}, { signal, timeoutMs: 15_000 });
+}
+
+export async function ensureMacosReady(
+	ctx: ExtensionContext,
+	state: PlatformReadyState,
+	signal?: AbortSignal,
+): Promise<PlatformReadyState> {
+	await macosHelper.ensureInstalled(signal);
+	if (!(await macosHelper.ensureDaemon(signal))) {
+		throw new Error(`tangu-computer-use helper app daemon did not start. Helper app: ${HELPER_APP_PATH}`);
+	}
+	const helperDiagnostics = await macosHelper.ensureProtocol(signal);
+	assertPlatformArchitecture("macOS", helperDiagnostics);
+
+	const now = Date.now();
+	const cachedStatus = state.permissionStatus;
+	const canUseCachedPermissions =
+		cachedStatus?.accessibility &&
+		cachedStatus.screenRecording &&
+		now - state.lastPermissionCheckAt < 2_000;
+	if (canUseCachedPermissions) {
+		return { ...state, helperDiagnostics };
+	}
+
+	let permissionStatus = await checkPermissions(signal);
+	let lastPermissionCheckAt = now;
+
+	if (!permissionStatus.accessibility || !permissionStatus.screenRecording) {
+		// Attribution "caller" means the helper is not running as the
+		// canonical installed app — grants would attach to the wrong identity.
+		const attributionHint = permissionStatus.source?.attribution === "caller"
+			? `Warning: the helper is not running as the installed tangu-computer-use.app (executable: ${permissionStatus.source?.executablePath ?? "unknown"}). Grants made now would attach to the launching app instead. Restart Tangu so the canonical helper is used.`
+			: undefined;
+		permissionStatus = await ensurePermissions(
+			ctx,
+			{
+				kinds: macosPermissionKinds,
+				copy: {
+					nonInteractiveError: (helperPath) => `tangu-computer-use setup requires an interactive session. Run 'tangu computer-use setup' in a terminal. ${GRANT_INSTRUCTIONS}\nHelper path: ${helperPath}`,
+					prompt: permissionPrompt,
+					incompleteError: (helperPath) => `tangu-computer-use setup is incomplete. ${GRANT_INSTRUCTIONS} Helper path: ${helperPath}`,
+					readyMessage: "tangu-computer-use is ready.",
+					stillMissing: missingPermissionMessage,
+				},
+				checkPermissions: (permissionSignal) => checkPermissions(permissionSignal ?? signal),
+				registerPermissions: (permissionSignal) => registerPermissions(permissionSignal ?? signal),
+				openPermissionPane: async (kind, permissionSignal) => {
+					await macosHelper.command("openPermissionPane", { kind }, { signal: permissionSignal ?? signal });
+				},
+				restartHelper: (permissionSignal) => macosHelper.restart(permissionSignal ?? signal),
+				permissionHint: attributionHint,
+			},
+			HELPER_APP_PATH,
+			signal,
+		);
+		lastPermissionCheckAt = Date.now();
+	}
+
+	return { permissionStatus, lastPermissionCheckAt, helperDiagnostics };
+}

package/src/vendor/platform/types.ts

@@ -0,0 +1,222 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { LookResponse } from "../outline.ts";
+import type { PermissionStatus } from "../permissions.ts";
+
+export type PlatformName = "macos" | "windows" | "linux";
+export type NativeInputDelivery = "hid" | "pid";
+export type ActOutcome = "worked" | "didnt" | "unknown";
+/**
+ * Best-effort presentation hint for a root. The seam guarantees only the
+ * `window` vs transient distinction; specific transient kinds are display hints
+ * and must never drive behavior in shared code. Platforms that need precise
+ * distinctions internally should use native signals.
+ */
+export type PlatformRootKind = "window" | "menu" | "sheet" | "popover" | "dialog";
+
+export interface PlatformDiagnostics {
+	protocolVersion: number;
+	architectureVersion?: number;
+	invariants?: string[];
+	pid: number;
+	parentPid?: number;
+	parentAppName?: string;
+	parentBundleId?: string;
+	parentPath?: string;
+	executablePath?: string;
+	os?: string;
+	arch?: string;
+	accessibility?: boolean;
+	screenRecording?: boolean;
+}
+
+export interface PlatformReadyState {
+	permissionStatus?: PermissionStatus;
+	lastPermissionCheckAt: number;
+	helperDiagnostics?: PlatformDiagnostics;
+}
+
+export interface PlatformRootQuery {
+	pid?: number;
+	title?: string;
+}
+
+export interface PlatformApp {
+	appName: string;
+	bundleId?: string;
+	pid: number;
+	isFrontmost?: boolean;
+}
+
+export interface FramePoints {
+	x: number;
+	y: number;
+	w: number;
+	h: number;
+}
+
+export interface PlatformRoot {
+	kind: PlatformRootKind;
+	rootRef?: string;
+	windowRef?: string;
+	windowId?: number;
+	pid?: number;
+	appName?: string;
+	bundleId?: string;
+	title: string;
+	role?: string;
+	subrole?: string;
+	zOrder: number;
+	framePoints: FramePoints;
+	scaleFactor: number;
+	isOnscreen: boolean;
+	isFocused: boolean;
+	isMinimized: boolean;
+	/** Best-effort; platforms without a main-window concept may mirror `isFocused`. */
+	isMain: boolean;
+	/** Platform-reported modality fact, including platform-specific modal/dialog/sheet signals. */
+	isModal: boolean;
+	metadata?: Record<string, unknown>;
+}
+
+export interface PlatformFrontmostResult {
+	appName: string;
+	bundleId?: string;
+	pid: number;
+	windowTitle?: string;
+	windowId?: number;
+	rootRef?: string;
+}
+
+export interface PlatformFocusWindowResult {
+	focused: boolean;
+	alreadyFocused?: boolean;
+	reason?: string;
+}
+
+export interface HelperActPerformed {
+	grounding?: "description" | "coordinates" | "keyboard-events";
+	/** `ax` means the platform accessibility API (AX on macOS, UIA on Windows). */
+	delivery?: "ax" | NativeInputDelivery;
+	refound?: boolean;
+	/** Free-form diagnostic naming the platform's delta mechanism. */
+	deltaSource?: string;
+	selectionGrounding?: "ax" | "keyboard";
+	transaction?: boolean;
+	actionCount?: number;
+	activated?: boolean;
+	raised?: boolean;
+	focused?: boolean;
+}
+
+export interface PlatformRootDelta {
+	change: "appeared" | "closed" | "focused";
+	kind: string;
+	ref?: string;
+	title?: string;
+	pid: number;
+	isModal?: boolean;
+	metadata?: Record<string, unknown>;
+}
+
+export interface HelperActResult {
+	outcome: ActOutcome;
+	performed?: HelperActPerformed;
+	evidence?: Record<string, unknown>;
+	error?: { code?: string; message?: string; whatIsThere?: unknown };
+	rootDelta?: PlatformRootDelta[];
+	steps?: HelperActResult[];
+	stoppedAt?: number;
+}
+
+export interface PlatformTarget {
+	pid?: number;
+	windowId?: number;
+	rootRef?: string;
+}
+
+export interface PlatformObserveRequest {
+	target: PlatformTarget;
+	/** Existing immutable look whose untouched refs/coordinate geometry survive a scoped refresh. */
+	baseLookId?: string;
+	readText: "auto" | "always" | "never";
+	scopeRef?: string;
+	maxDimension?: number;
+	includeImage?: boolean;
+}
+
+export type PlatformActTarget = { ref: string } | { x: number; y: number } | { focus: PlatformPoint };
+type PlatformDeliveryPolicy = "ax_only" | "background" | "default" | "foreground";
+type PlatformMouseButton = "left" | "right" | "middle";
+type PlatformActDeliveryParam = { delivery?: NativeInputDelivery };
+export type PlatformPoint = { x: number; y: number };
+
+export interface PlatformActRequestBase {
+	lookId: string;
+	pid?: number;
+	target: PlatformActTarget;
+	policy: PlatformDeliveryPolicy;
+}
+
+export type PlatformActRequest = PlatformActRequestBase & (
+	| { action: "press" | "click"; params: { button?: PlatformMouseButton; clickCount?: number } & PlatformActDeliveryParam }
+	| { action: "setText"; params: { text: string } & PlatformActDeliveryParam }
+	| { action: "typeText"; params: { text: string } & PlatformActDeliveryParam }
+	| { action: "keypress"; params: { keys: string[] } & PlatformActDeliveryParam }
+	| { action: "scroll"; params: { scrollX: number; scrollY: number } & PlatformActDeliveryParam }
+	| { action: "drag"; params: { path: PlatformPoint[] } & PlatformActDeliveryParam }
+	| { action: "moveMouse"; params: PlatformActDeliveryParam }
+);
+
+export interface PlatformReadTextRequest {
+	/** Observation that owns the element ref. Native backends must not resolve across observations. */
+	lookId: string;
+	elementRef: string;
+	offset: number;
+	limit: number;
+}
+
+export interface PlatformReadTextResponse {
+	text: string;
+	offset: number;
+	limit: number;
+	totalChars: number;
+	hasMore: boolean;
+}
+
+export interface PlatformWaitForRequest extends PlatformTarget {
+	lookId?: string;
+	text?: string;
+	role?: string;
+	value?: string;
+	scopeRef?: string;
+	scopeExact?: boolean;
+	gone: boolean;
+	timeoutMs: number;
+}
+
+export interface PlatformWaitForResponse {
+	found: boolean;
+	gone?: boolean;
+	timedOut?: boolean;
+	nodeCount?: number;
+}
+
+export interface ComputerUsePlatformBackend {
+	name: PlatformName;
+	/** Release process-local resources when the Pi session is torn down. */
+	shutdown?(): void | Promise<void>;
+	ensureReady(ctx: ExtensionContext, state: PlatformReadyState, signal?: AbortSignal): Promise<PlatformReadyState>;
+	listApps(signal?: AbortSignal): Promise<PlatformApp[]>;
+	listRoots(query: PlatformRootQuery, signal?: AbortSignal): Promise<PlatformRoot[]>;
+	getFrontmost(signal?: AbortSignal): Promise<PlatformFrontmostResult>;
+	focusWindow(target: PlatformTarget, signal?: AbortSignal): Promise<PlatformFocusWindowResult>;
+	observe(request: PlatformObserveRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<LookResponse>;
+	act(request: PlatformActRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<HelperActResult>;
+	/** Execute one-resource actions with one root baseline and one final settle. */
+	actBatch?(requests: PlatformActRequest[], options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<HelperActResult>;
+	readText(args: PlatformReadTextRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<PlatformReadTextResponse>;
+	waitFor(args: PlatformWaitForRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<PlatformWaitForResponse>;
+	isBrowserApp(appName: string, bundleId?: string): boolean;
+	isChromeFamilyApp(appName: string, bundleId?: string): boolean;
+	openBrowserLocation(target: { appName: string; bundleId?: string }, url: string, signal?: AbortSignal): Promise<boolean>;
+}