@forsion/tangu-computer-use 0.5.8

package/src/vendor/platform/linux/helper.ts

@@ -0,0 +1,238 @@
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { constants as fsConstants } from "node:fs";
+import { access } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
+const SETUP_HELPER_SCRIPT = path.join(PACKAGE_ROOT, "scripts", "setup-helper.mjs");
+const HELPER_SETUP_TIMEOUT_MS = 60_000;
+const COMMAND_TIMEOUT_MS = 15_000;
+
+export const LINUX_HELPER_PROTOCOL_VERSION = 4;
+export const LINUX_HELPER_PATH = process.env.PI_COMPUTER_USE_LINUX_HELPER_PATH
+	|| path.join(os.homedir(), ".pi", "agent", "helpers", "tangu-computer-use", "linux-bridge");
+
+interface Pending<T> {
+	resolve(value: T): void;
+	reject(error: Error): void;
+	timer: NodeJS.Timeout;
+}
+
+async function isExecutable(filePath: string): Promise<boolean> {
+	try { await access(filePath, fsConstants.X_OK); return true; } catch { return false; }
+}
+
+async function runProcess(command: string, args: string[], timeoutMs: number, signal?: AbortSignal, env?: NodeJS.ProcessEnv): Promise<void> {
+	if (signal?.aborted) throw new Error("Operation aborted.");
+	await new Promise<void>((resolve, reject) => {
+		const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], env });
+		let stdout = "";
+		let stderr = "";
+		let settled = false;
+		const finish = (error?: Error) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			signal?.removeEventListener("abort", onAbort);
+			if (error) reject(error); else resolve();
+		};
+		const timer = setTimeout(() => {
+			child.kill("SIGTERM");
+			finish(new Error(`Command timed out after ${timeoutMs}ms: ${command} ${args.join(" ")}`));
+		}, timeoutMs);
+		const onAbort = () => { child.kill("SIGTERM"); finish(new Error("Operation aborted.")); };
+		child.stdout.on("data", (chunk) => { stdout += String(chunk); });
+		child.stderr.on("data", (chunk) => { stderr += String(chunk); });
+		child.on("error", (error) => finish(error));
+		child.on("close", (code) => {
+			if (code === 0) return finish();
+			const output = [stderr.trim(), stdout.trim()].filter(Boolean).join("\n");
+			finish(new Error(`Command failed (${code}): ${command} ${args.join(" ")}\n${output}`.trim()));
+		});
+		signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}
+
+async function waitForShared<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
+	if (!signal) return await promise;
+	if (signal.aborted) throw new Error("Operation aborted.");
+	return await new Promise<T>((resolve, reject) => {
+		const onAbort = () => reject(new Error("Operation aborted."));
+		signal.addEventListener("abort", onAbort, { once: true });
+		promise.then(
+			(value) => {
+				signal.removeEventListener("abort", onAbort);
+				resolve(value);
+			},
+			(error) => {
+				signal.removeEventListener("abort", onAbort);
+				reject(error);
+			},
+		);
+	});
+}
+
+export interface LinuxHelperClientOptions {
+	helperPath?: string;
+	setupHelperScript?: string;
+}
+
+export class LinuxHelperClient {
+	private installChecked = false;
+	private installPromise?: Promise<void>;
+	private child?: ChildProcessWithoutNullStreams;
+	private processPromise?: Promise<ChildProcessWithoutNullStreams>;
+	private buffer = "";
+	private pending = new Map<string, Pending<unknown>>();
+	private readonly helperPath: string;
+	private readonly setupHelperScript: string;
+
+	constructor(options: LinuxHelperClientOptions = {}) {
+		this.helperPath = options.helperPath ?? LINUX_HELPER_PATH;
+		this.setupHelperScript = options.setupHelperScript ?? SETUP_HELPER_SCRIPT;
+	}
+
+	dispose(): void {
+		this.rejectPending(new Error("Linux helper closed because the Tangu session ended."));
+		const child = this.child;
+		this.child = undefined;
+		if (!child) return;
+		child.stdin.destroy();
+		child.stdout.destroy();
+		child.stderr.destroy();
+		child.kill("SIGTERM");
+		child.unref();
+	}
+
+	private rejectPending(error: Error): void {
+		for (const pending of this.pending.values()) {
+			clearTimeout(pending.timer);
+			pending.reject(error);
+		}
+		this.pending.clear();
+		this.buffer = "";
+	}
+
+	async ensureInstalled(signal?: AbortSignal): Promise<void> {
+		if ((await isExecutable(this.helperPath)) && this.installChecked) return;
+		if (!this.installPromise) {
+			const installPromise = (async () => {
+				await runProcess(process.execPath, [this.setupHelperScript, "--platform", "linux", "--runtime"], HELPER_SETUP_TIMEOUT_MS, undefined, {
+					...process.env,
+					ELECTRON_RUN_AS_NODE: "1",
+					BUN_BE_BUN: "1",
+					PI_COMPUTER_USE_LINUX_HELPER_PATH: this.helperPath,
+				});
+				if (!(await isExecutable(this.helperPath))) throw new Error(`Failed to install Linux helper at ${this.helperPath}.`);
+				this.installChecked = true;
+			})();
+			this.installPromise = installPromise;
+			installPromise.then(
+				() => { if (this.installPromise === installPromise) this.installPromise = undefined; },
+				() => { if (this.installPromise === installPromise) this.installPromise = undefined; },
+			);
+		}
+		await waitForShared(this.installPromise, signal);
+	}
+
+	private async process(signal?: AbortSignal): Promise<ChildProcessWithoutNullStreams> {
+		await this.ensureInstalled(signal);
+		if (this.processPromise) return await waitForShared(this.processPromise, signal);
+		if (this.child && this.child.exitCode === null && !this.child.killed) return this.child;
+		if (!this.processPromise) {
+			const processPromise = this.startProcess();
+			this.processPromise = processPromise;
+			processPromise.then(
+				() => { if (this.processPromise === processPromise) this.processPromise = undefined; },
+				() => { if (this.processPromise === processPromise) this.processPromise = undefined; },
+			);
+		}
+		return await waitForShared(this.processPromise, signal);
+	}
+
+	private async startProcess(): Promise<ChildProcessWithoutNullStreams> {
+		const child = spawn(this.helperPath, [], { stdio: ["pipe", "pipe", "pipe"] });
+		child.stdout.setEncoding("utf8");
+		child.stderr.setEncoding("utf8");
+		child.stdin.setDefaultEncoding("utf8");
+		child.stdout.on("data", (chunk: string) => this.onStdout(chunk));
+		child.on("exit", (code, signalName) => {
+			if (this.child !== child) return;
+			this.child = undefined;
+			this.rejectPending(new Error(`Linux helper exited${signalName ? ` on ${signalName}` : ` with code ${code ?? "unknown"}`}.`));
+		});
+		child.on("error", (error) => {
+			if (this.child !== child) return;
+			this.child = undefined;
+			this.rejectPending(error);
+		});
+		this.child = child;
+		this.buffer = "";
+		return await new Promise<ChildProcessWithoutNullStreams>((resolve, reject) => {
+			child.once("spawn", () => resolve(child));
+			child.once("error", reject);
+		});
+	}
+
+	private onStdout(chunk: string): void {
+		this.buffer += chunk;
+		for (;;) {
+			const newline = this.buffer.indexOf("\n");
+			if (newline < 0) return;
+			const line = this.buffer.slice(0, newline).trim();
+			this.buffer = this.buffer.slice(newline + 1);
+			if (!line) continue;
+			let parsed: any;
+			try { parsed = JSON.parse(line); } catch { continue; }
+			const pending = this.pending.get(parsed.id);
+			if (!pending) continue;
+			this.pending.delete(parsed.id);
+			clearTimeout(pending.timer);
+			if (parsed.protocolVersion !== LINUX_HELPER_PROTOCOL_VERSION) {
+				pending.reject(new Error(`Linux helper protocol mismatch: expected ${LINUX_HELPER_PROTOCOL_VERSION}, got ${parsed.protocolVersion ?? "unknown"}. Restart Tangu to use the installed helper.`));
+			} else if (parsed.ok === true) {
+				pending.resolve(parsed.result);
+			} else {
+				const error = new Error(parsed.error?.message ?? "Linux helper command failed.") as Error & { code?: string };
+				error.code = parsed.error?.code;
+				pending.reject(error);
+			}
+		}
+	}
+
+	async command<T>(cmd: string, args: Record<string, unknown> = {}, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<T> {
+		const child = await this.process(options?.signal);
+		const id = randomUUID();
+		const timeoutMs = options?.timeoutMs ?? COMMAND_TIMEOUT_MS;
+		return await new Promise<T>((resolve, reject) => {
+			const onAbort = () => {
+				this.pending.delete(id);
+				clearTimeout(timer);
+				reject(new Error("Operation aborted."));
+			};
+			const timer = setTimeout(() => {
+				options?.signal?.removeEventListener("abort", onAbort);
+				this.pending.delete(id);
+				reject(new Error(`Helper command '${cmd}' timed out after ${timeoutMs}ms.`));
+			}, timeoutMs);
+			this.pending.set(id, {
+				resolve: (value) => { options?.signal?.removeEventListener("abort", onAbort); resolve(value as T); },
+				reject: (error) => { options?.signal?.removeEventListener("abort", onAbort); reject(error); },
+				timer,
+			});
+			options?.signal?.addEventListener("abort", onAbort, { once: true });
+			child.stdin.write(`${JSON.stringify({ protocolVersion: LINUX_HELPER_PROTOCOL_VERSION, id, cmd, args })}\n`, (error) => {
+				if (!error) return;
+				options?.signal?.removeEventListener("abort", onAbort);
+				this.pending.delete(id);
+				clearTimeout(timer);
+				reject(error);
+			});
+		});
+	}
+}
+
+export const linuxHelper = new LinuxHelperClient();

package/src/vendor/platform/macos/backend.ts

@@ -0,0 +1,131 @@
+import { getComputerUseConfig } from "../../config.ts";
+import { parseLookResponse, type LookResponse } from "../../outline.ts";
+import { toBoolean, toFiniteNumber, toOptionalString } from "../coerce.ts";
+import type { ComputerUsePlatformBackend, FramePoints, HelperActResult, PlatformActRequest, PlatformApp, PlatformFocusWindowResult, PlatformFrontmostResult, PlatformObserveRequest, PlatformReadTextRequest, PlatformReadTextResponse, PlatformRoot, PlatformRootKind, PlatformRootQuery, PlatformTarget, PlatformWaitForRequest, PlatformWaitForResponse } from "../types.ts";
+import { macosHelper } from "./helper.ts";
+
+function parseApps(result: unknown): PlatformApp[] {
+	const array = Array.isArray(result) ? result : (result as any)?.apps;
+	if (!Array.isArray(array)) return [];
+
+	return array
+		.map((raw) => {
+			const pid = Math.trunc(toFiniteNumber((raw as any)?.pid, NaN));
+			if (!Number.isFinite(pid) || pid <= 0) return undefined;
+			const appName = toOptionalString((raw as any)?.appName) ?? "Unknown App";
+			return {
+				appName,
+				bundleId: toOptionalString((raw as any)?.bundleId),
+				pid,
+				isFrontmost: toBoolean((raw as any)?.isFrontmost),
+			} as PlatformApp;
+		})
+		.filter((item): item is PlatformApp => Boolean(item));
+}
+
+function parseFramePoints(raw: unknown): FramePoints {
+	const frame = (raw as any)?.framePoints ?? {};
+	return {
+		x: toFiniteNumber(frame.x, 0),
+		y: toFiniteNumber(frame.y, 0),
+		w: Math.max(1, toFiniteNumber(frame.w, 1)),
+		h: Math.max(1, toFiniteNumber(frame.h, 1)),
+	};
+}
+
+function parseRoots(result: unknown): PlatformRoot[] {
+	const array = Array.isArray(result) ? result : (result as any)?.roots;
+	if (!Array.isArray(array)) return [];
+
+	return array.map((raw) => {
+		const metadata = typeof (raw as any)?.metadata === "object" && (raw as any).metadata !== null ? (raw as any).metadata as Record<string, unknown> : {};
+		const kind = ["window", "menu", "sheet", "popover", "dialog"].includes((raw as any)?.kind) ? (raw as any).kind as PlatformRootKind : "window";
+		return {
+			kind,
+			rootRef: toOptionalString((raw as any)?.rootRef ?? (raw as any)?.windowRef),
+			windowRef: toOptionalString((raw as any)?.windowRef ?? (raw as any)?.rootRef),
+			windowId: Number.isFinite((raw as any)?.windowId) ? Math.trunc((raw as any).windowId) : undefined,
+			pid: Number.isFinite((raw as any)?.pid) ? Math.trunc((raw as any).pid) : undefined,
+			appName: toOptionalString((raw as any)?.appName),
+			bundleId: toOptionalString((raw as any)?.bundleId),
+			title: toOptionalString((raw as any)?.title) ?? "",
+			role: toOptionalString((raw as any)?.role),
+			subrole: toOptionalString((raw as any)?.subrole),
+			framePoints: parseFramePoints(raw),
+			scaleFactor: Math.max(1, toFiniteNumber((raw as any)?.scaleFactor, 1)),
+			zOrder: Math.trunc(toFiniteNumber((raw as any)?.zOrder, 0)),
+			isMinimized: toBoolean((raw as any)?.isMinimized),
+			isOnscreen: toBoolean((raw as any)?.isOnscreen),
+			isMain: toBoolean((raw as any)?.isMain),
+			isFocused: toBoolean((raw as any)?.isFocused),
+			isModal: toBoolean((raw as any)?.isModal),
+			metadata,
+		};
+	});
+}
+
+function helperAction(request: PlatformActRequest): Record<string, unknown> {
+	if (!("focus" in request.target)) return { ...request };
+	return { ...request, target: request.target.focus, params: { ...request.params, preserveFocus: true } };
+}
+
+export const macosBackend: Pick<ComputerUsePlatformBackend, "listApps" | "listRoots" | "getFrontmost" | "focusWindow" | "observe" | "act" | "actBatch" | "readText" | "waitFor"> = {
+	async listApps(signal?: AbortSignal): Promise<PlatformApp[]> {
+		return parseApps(await macosHelper.command<unknown>("listApps", {}, { signal }));
+	},
+
+	async listRoots(query: PlatformRootQuery, signal?: AbortSignal): Promise<PlatformRoot[]> {
+		return parseRoots(await macosHelper.command<unknown>("listRoots", {
+			...(Number.isFinite(query.pid) ? { pid: Math.trunc(query.pid!) } : {}),
+			...(query.title?.trim() ? { title: query.title.trim() } : {}),
+		}, { signal }));
+	},
+
+	async getFrontmost(signal?: AbortSignal): Promise<PlatformFrontmostResult> {
+		const result = await macosHelper.command<any>("getFrontmost", {}, { signal });
+		const pid = Math.trunc(toFiniteNumber(result?.pid, NaN));
+		if (!Number.isFinite(pid) || pid <= 0) {
+			throw new Error("No frontmost app was available for screenshot targeting.");
+		}
+		return {
+			appName: toOptionalString(result?.appName) ?? "Unknown App",
+			bundleId: toOptionalString(result?.bundleId),
+			pid,
+			windowTitle: toOptionalString(result?.windowTitle),
+			windowId: Number.isFinite(result?.windowId) ? Math.trunc(result.windowId) : undefined,
+		};
+	},
+
+	async focusWindow(target: PlatformTarget, signal?: AbortSignal): Promise<PlatformFocusWindowResult> {
+		return await macosHelper.command<PlatformFocusWindowResult>("focusWindow", { ...target }, { signal });
+	},
+
+	async observe(request: PlatformObserveRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<LookResponse> {
+		return parseLookResponse(await macosHelper.command("look", {
+			baseLookId: request.baseLookId,
+			windowId: request.target.windowId,
+			windowRef: request.target.rootRef,
+			maxDimension: request.maxDimension,
+			readText: request.readText,
+			scopeRef: request.scopeRef,
+			includeImage: request.includeImage,
+		}, options));
+	},
+
+	async act(request: PlatformActRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<HelperActResult> {
+		return await macosHelper.command<HelperActResult>("act", { ...helperAction(request), cursorOverlay: getComputerUseConfig().cursor_overlay }, options);
+	},
+
+	async actBatch(requests: PlatformActRequest[], options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<HelperActResult> {
+		const cursorOverlay = getComputerUseConfig().cursor_overlay;
+		return await macosHelper.command<HelperActResult>("actBatch", { actions: requests.map((request) => ({ ...helperAction(request), cursorOverlay })) }, options);
+	},
+
+	async readText(args: PlatformReadTextRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<PlatformReadTextResponse> {
+		return await macosHelper.command("axReadText", { ...args }, options);
+	},
+
+	async waitFor(args: PlatformWaitForRequest, options?: { timeoutMs?: number; signal?: AbortSignal }): Promise<PlatformWaitForResponse> {
+		return await macosHelper.command("axWaitFor", { ...args }, options);
+	},
+};

package/src/vendor/platform/macos/browser.ts

@@ -0,0 +1,110 @@
+import { runProcess } from "./helper.ts";
+
+const BROWSER_WINDOW_OPEN_TIMEOUT_MS = 10_000;
+
+const BROWSER_BUNDLE_IDS = new Set([
+	"com.apple.Safari",
+	"com.google.Chrome",
+	"org.chromium.Chromium",
+	"company.thebrowser.Browser",
+	"com.brave.Browser",
+	"com.microsoft.edgemac",
+	"com.vivaldi.Vivaldi",
+	"net.imput.helium",
+	"org.mozilla.firefox",
+]);
+const BROWSER_APP_NAMES = new Set([
+	"safari",
+	"google chrome",
+	"chrome",
+	"chromium",
+	"arc",
+	"brave browser",
+	"brave",
+	"microsoft edge",
+	"edge",
+	"vivaldi",
+	"helium",
+	"firefox",
+]);
+const CHROME_FAMILY_BUNDLE_IDS = new Set([
+	"com.google.Chrome",
+	"org.chromium.Chromium",
+	"company.thebrowser.Browser",
+	"com.brave.Browser",
+	"com.microsoft.edgemac",
+	"com.vivaldi.Vivaldi",
+	"net.imput.helium",
+]);
+const CHROME_FAMILY_APP_NAMES = new Set([
+	"google chrome",
+	"chrome",
+	"chromium",
+	"arc",
+	"brave browser",
+	"brave",
+	"microsoft edge",
+	"edge",
+	"vivaldi",
+	"helium",
+]);
+
+function normalizeText(value: string): string {
+	return value.trim().toLowerCase();
+}
+
+function escapeAppleScriptString(value: string): string {
+	return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
+}
+
+function appendBrowserJavaScriptAppleEventsHint(error: Error): Error {
+	const hint = [
+		"Browser JavaScript Apple Events are disabled for the target browser.",
+		"Ask the user to enable \"Allow JavaScript from Apple Events\" in the browser's developer menu, then retry the browser action.",
+	].join(" ");
+	if (!/not allowed to send javascript commands|executing javascript through applescript is turned off|allow javascript from apple events|enable javascript from apple events/i.test(error.message) || error.message.includes(hint)) {
+		return error;
+	}
+	const enhanced = new Error(`${error.message}\n\n${hint}`);
+	enhanced.name = error.name;
+	return enhanced;
+}
+
+export function isBrowserApp(appName: string, bundleId?: string): boolean {
+	return BROWSER_BUNDLE_IDS.has(bundleId ?? "") || BROWSER_APP_NAMES.has(normalizeText(appName));
+}
+
+export function isChromeFamilyApp(appName: string, bundleId?: string): boolean {
+	return CHROME_FAMILY_BUNDLE_IDS.has(bundleId ?? "") || CHROME_FAMILY_APP_NAMES.has(normalizeText(appName));
+}
+
+function scriptForBrowserOpenLocation(target: { appName: string; bundleId?: string }, url: string): string[] | undefined {
+	const appTarget = target.bundleId
+		? `application id "${escapeAppleScriptString(target.bundleId)}"`
+		: `application "${escapeAppleScriptString(target.appName)}"`;
+	const escapedUrl = escapeAppleScriptString(url);
+	const normalizedName = normalizeText(target.appName);
+	if (target.bundleId === "com.apple.Safari" || normalizedName === "safari") {
+		return [`tell ${appTarget} to set URL of front document to "${escapedUrl}"`];
+	}
+	if (isChromeFamilyApp(target.appName, target.bundleId)) {
+		return [`tell ${appTarget} to set URL of active tab of front window to "${escapedUrl}"`];
+	}
+	return undefined;
+}
+
+export async function openBrowserLocationWithAppleScript(
+	target: { appName: string; bundleId?: string },
+	url: string,
+	signal?: AbortSignal,
+): Promise<boolean> {
+	const script = scriptForBrowserOpenLocation(target, url);
+	if (!script) return false;
+	const args = script.flatMap((line) => ["-e", line]);
+	try {
+		await runProcess("osascript", args, BROWSER_WINDOW_OPEN_TIMEOUT_MS, signal);
+		return true;
+	} catch (error) {
+		throw appendBrowserJavaScriptAppleEventsHint(error instanceof Error ? error : new Error(String(error)));
+	}
+}

package/src/vendor/platform/macos/helper-path.d.mts

@@ -0,0 +1,9 @@
+export interface MacosHelperPathOptions {
+	env?: NodeJS.ProcessEnv;
+	homeDir?: string;
+	systemHelperAppPath?: string;
+	fileExists?: (filePath: string) => boolean;
+	directoryIsWritable?: (directoryPath: string) => boolean;
+}
+
+export function resolveMacosHelperAppPath(options?: MacosHelperPathOptions): string;

package/src/vendor/platform/macos/helper-path.mjs

@@ -0,0 +1,34 @@
+import { accessSync, constants as fsConstants, existsSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const HELPER_APP_NAME = "tangu-computer-use.app";
+const SYSTEM_HELPER_APP_PATH = path.join("/Applications", HELPER_APP_NAME);
+
+/**
+ * Resolve one stable helper location for both installation and runtime use.
+ * Existing writable system installs stay in place; all other installs are
+ * per-user so standard macOS accounts never require administrator access.
+ */
+export function resolveMacosHelperAppPath(options = {}) {
+	const env = options.env ?? process.env;
+	const explicitPath = env.PI_COMPUTER_USE_HELPER_APP_PATH?.trim();
+	if (explicitPath) return path.resolve(explicitPath);
+
+	const homeDir = options.homeDir ?? os.homedir();
+	const systemHelperAppPath = options.systemHelperAppPath ?? SYSTEM_HELPER_APP_PATH;
+	const fileExists = options.fileExists ?? existsSync;
+	const directoryIsWritable = options.directoryIsWritable ?? ((directoryPath) => {
+		try {
+			accessSync(directoryPath, fsConstants.W_OK);
+			return true;
+		} catch {
+			return false;
+		}
+	});
+
+	if (fileExists(systemHelperAppPath) && directoryIsWritable(path.dirname(systemHelperAppPath))) {
+		return systemHelperAppPath;
+	}
+	return path.join(homeDir, "Applications", HELPER_APP_NAME);
+}