@forsion/tangu-computer-use 0.5.8

package/src/vendor/config.ts

@@ -0,0 +1,113 @@
+import { existsSync, readFileSync } from "node:fs";
+import path from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+export interface ComputerUseConfig {
+	browser_use: boolean;
+	headless: boolean;
+	cursor_overlay: boolean;
+	managed_browser: "helium" | "chrome";
+}
+
+export interface ComputerUseConfigSource {
+	path: string;
+	exists: boolean;
+	values?: Partial<ComputerUseConfig>;
+	error?: string;
+}
+
+export interface LoadedComputerUseConfig {
+	config: ComputerUseConfig;
+	sources: ComputerUseConfigSource[];
+	env: Partial<ComputerUseConfig>;
+}
+
+const DEFAULT_CONFIG: ComputerUseConfig = {
+	browser_use: true,
+	headless: false,
+	cursor_overlay: true,
+	managed_browser: "chrome",
+};
+
+let activeConfig: ComputerUseConfig = { ...DEFAULT_CONFIG };
+let activeLoadedConfig: LoadedComputerUseConfig = { config: activeConfig, sources: [], env: {} };
+
+function parseBoolean(value: unknown): boolean | undefined {
+	if (typeof value === "boolean") return value;
+	if (typeof value === "number") return value === 1 ? true : value === 0 ? false : undefined;
+	if (typeof value !== "string") return undefined;
+	const normalized = value.trim().toLowerCase();
+	if (["1", "true", "yes", "on", "enabled"].includes(normalized)) return true;
+	if (["0", "false", "no", "off", "disabled"].includes(normalized)) return false;
+	return undefined;
+}
+
+function normalizePartial(raw: unknown): Partial<ComputerUseConfig> {
+	if (!raw || typeof raw !== "object") return {};
+	const source = (raw as any).computer_use && typeof (raw as any).computer_use === "object" ? (raw as any).computer_use : raw;
+	const out: Partial<ComputerUseConfig> = {};
+	const browserUse = parseBoolean((source as any).browser_use);
+	const headless = parseBoolean((source as any).headless);
+	const cursorOverlay = parseBoolean((source as any).cursor_overlay);
+	if (browserUse !== undefined) out.browser_use = browserUse;
+	if (headless !== undefined) out.headless = headless;
+	if (cursorOverlay !== undefined) out.cursor_overlay = cursorOverlay;
+	const managedBrowser = (source as any).managed_browser;
+	if (managedBrowser === "helium" || managedBrowser === "chrome") out.managed_browser = managedBrowser;
+	return out;
+}
+
+function readConfigFile(filePath: string): ComputerUseConfigSource {
+	if (!existsSync(filePath)) return { path: filePath, exists: false };
+	try {
+		const parsed = JSON.parse(readFileSync(filePath, "utf-8"));
+		return { path: filePath, exists: true, values: normalizePartial(parsed) };
+	} catch (error) {
+		return { path: filePath, exists: true, error: error instanceof Error ? error.message : String(error) };
+	}
+}
+
+function readEnv(): Partial<ComputerUseConfig> {
+	const out: Partial<ComputerUseConfig> = {};
+	const browserUse = parseBoolean(process.env.PI_COMPUTER_USE_BROWSER_USE);
+	const headless = parseBoolean(process.env.PI_COMPUTER_USE_HEADLESS);
+	const cursorOverlay = parseBoolean(process.env.PI_COMPUTER_USE_CURSOR_OVERLAY);
+	if (browserUse !== undefined) out.browser_use = browserUse;
+	if (headless !== undefined) out.headless = headless;
+	if (cursorOverlay !== undefined) out.cursor_overlay = cursorOverlay;
+	const managedBrowser = process.env.PI_COMPUTER_USE_MANAGED_BROWSER;
+	if (managedBrowser === "helium" || managedBrowser === "chrome") out.managed_browser = managedBrowser;
+	return out;
+}
+
+export function loadComputerUseConfig(cwd: string): LoadedComputerUseConfig {
+	const sources = [
+		readConfigFile(path.join(getAgentDir(), "extensions", "tangu-computer-use.json")),
+		readConfigFile(path.join(cwd, ".pi", "computer-use.json")),
+	];
+	const env = readEnv();
+	const config = { ...DEFAULT_CONFIG };
+	for (const source of sources) {
+		if (source.values) Object.assign(config, source.values);
+	}
+	Object.assign(config, env);
+	activeConfig = config;
+	activeLoadedConfig = { config, sources, env };
+	return activeLoadedConfig;
+}
+
+export function getComputerUseConfig(): ComputerUseConfig {
+	return activeConfig;
+}
+
+export function getLoadedComputerUseConfig(): LoadedComputerUseConfig {
+	return activeLoadedConfig;
+}
+
+export function isHeadlessMode(): boolean {
+	return activeConfig.headless;
+}
+
+export function isBrowserUseEnabled(): boolean {
+	return activeConfig.browser_use;
+}

package/src/vendor/contract.ts

@@ -0,0 +1,104 @@
+export type RootSelector = string;
+export type ImageMode = "auto" | "always" | "never";
+export type MouseButtonName = "left" | "right" | "middle";
+
+export interface ObserveTargetParams {
+	root?: RootSelector;
+}
+
+export interface FindParams {
+	text?: string;
+	app?: string;
+	bundleId?: string;
+	pid?: number;
+	/** Filters on the platform's best-effort presentation hint; only window vs transient is guaranteed. */
+	kind?: "window" | "menu" | "sheet" | "popover" | "dialog" | "browser_page";
+}
+
+export interface StateTargetParams {
+	stateId?: string;
+}
+
+export interface NavigateBrowserParams extends StateTargetParams {
+	url: string;
+}
+
+export interface LaunchBrowserParams {
+	url?: string;
+}
+
+export interface EvaluateBrowserParams {
+	stateId: string;
+	expression: string;
+}
+
+export interface ObserveParams extends ObserveTargetParams {
+	mode?: "semantic" | "visual" | "fused";
+	/** Internal capture override; not part of the model-facing schema. */
+	readText?: "auto" | "always" | "never";
+}
+
+export interface SearchUiParams extends StateTargetParams {
+	text?: string;
+	role?: string;
+	capability?: string;
+}
+
+export interface ExpandUiParams extends StateTargetParams {
+	ref: string;
+	depth?: number;
+}
+
+export interface InspectUiParams extends StateTargetParams {
+	ref: string;
+}
+
+export interface UiCondition {
+	ref?: string;
+	scopeRef?: string;
+	text?: string;
+	role?: string;
+	value?: string;
+	until?: "present" | "absent";
+	timeoutMs?: number;
+}
+
+export interface UiAction {
+	action: "press" | "click" | "setText" | "typeText" | "keypress" | "scroll" | "drag" | "moveMouse";
+	ref?: string;
+	x?: number;
+	y?: number;
+	text?: string;
+	keys?: string[];
+	scrollX?: number;
+	scrollY?: number;
+	path?: Array<{ x: number; y: number } | [number, number]>;
+	button?: MouseButtonName;
+	clickCount?: number;
+}
+
+export interface ActParams extends StateTargetParams {
+	actions: UiAction[];
+	expect?: UiCondition;
+}
+
+export interface ReadTextParams extends StateTargetParams {
+	ref: string;
+	offset?: number;
+}
+
+export interface WaitForParams extends StateTargetParams, UiCondition {}
+
+export const AGENT_TOOL_NAMES = new Set([
+	"find_roots",
+	"read_text",
+	"wait_for",
+	"observe_ui",
+	"search_ui",
+	"expand_ui",
+	"inspect_ui",
+	"act_ui",
+	"navigate_browser",
+	"evaluate_browser",
+	"launch_browser",
+]);

package/src/vendor/note.ts

@@ -0,0 +1,195 @@
+import { nodeByRef, outlineNodeLabel, type Outline, type OutlineNode } from "./outline.ts";
+
+interface NoteRegion {
+	key: string;
+	label: string;
+	status: "seen" | "changed" | "never-looked";
+	detail?: string;
+}
+
+export interface WindowNote {
+	windowRef: string;
+	title: string;
+	pairing: "exact" | "high" | "low";
+	lastLookId?: string;
+	regions: NoteRegion[];
+}
+
+interface NoteWindowInput {
+	windowRef?: string;
+	title: string;
+	pairing?: "exact" | "high" | "low";
+	pairingScore?: number;
+}
+
+interface NoteRefreshOutcome {
+	window: NoteWindowInput;
+	windowChanged?: boolean;
+	newWindowLabel?: string;
+	rootDelta?: Array<{ change: string; kind: string; title?: string; ref?: string }>;
+}
+
+function normalizedLabel(value: string): string {
+	return value.toLowerCase().replace(/\s+/g, " ").trim();
+}
+
+function nodeLabel(node: OutlineNode): string {
+	return outlineNodeLabel(node) || node.role || node.ref;
+}
+
+function regionKey(node: OutlineNode): string {
+	return `${node.role || "AXUnknown"}:${node.identifier || normalizedLabel(nodeLabel(node)) || node.ref}`;
+}
+
+function regionLabel(node: OutlineNode): string {
+	const label = nodeLabel(node);
+	return normalizedLabel(label) || normalizedLabel(node.role) || node.ref;
+}
+
+function topLevelRegions(outline: Outline): Array<{ node: OutlineNode; key: string; label: string }> {
+	const nodes = outline.root.children.length ? outline.root.children : [outline.root];
+	return nodes.map((node) => ({ node, key: regionKey(node), label: regionLabel(node) }));
+}
+
+function topLevelAncestor(node: OutlineNode, outline: Outline): OutlineNode {
+	let current = node;
+	while (current.parent && current.parent !== outline.root) current = current.parent;
+	return current;
+}
+
+function uniqueRegions(regions: NoteRegion[]): NoteRegion[] {
+	const seen = new Set<string>();
+	const output: NoteRegion[] = [];
+	for (const region of regions) {
+		const dedupeKey = `${region.key}:${region.status}:${region.detail ?? ""}`;
+		if (seen.has(dedupeKey)) continue;
+		seen.add(dedupeKey);
+		output.push(region);
+	}
+	return output;
+}
+
+function frontierRegions(outline: Outline, topLevels: Array<{ node: OutlineNode; key: string; label: string }>): NoteRegion[] {
+	const output: NoteRegion[] = [];
+	for (const node of outline.nodes) {
+		if (node.scrollExtent && node.scrollExtent.seen < node.scrollExtent.total) {
+			const ancestor = topLevelAncestor(node, outline);
+			const top = topLevels.find((candidate) => candidate.node === ancestor);
+			output.push({
+				key: `${top?.key ?? regionKey(ancestor)}:scroll:${node.ref}`,
+				label: top?.label ?? regionLabel(ancestor),
+				status: "never-looked",
+				detail: `${node.scrollExtent.total} rows, ${node.scrollExtent.seen} seen`,
+			});
+		}
+		if (node.truncated) {
+			const ancestor = topLevelAncestor(node, outline);
+			const top = topLevels.find((candidate) => candidate.node === ancestor);
+			output.push({
+				key: `${top?.key ?? regionKey(ancestor)}:truncated:${node.ref}`,
+				label: top?.label ?? regionLabel(ancestor),
+				status: "never-looked",
+				detail: "subtree not walked",
+			});
+		}
+	}
+	return output;
+}
+
+export function noteFromLook(prev: WindowNote | undefined, outline: Outline, window: NoteWindowInput): WindowNote {
+	const topLevels = topLevelRegions(outline);
+	const currentKeys = new Set(topLevels.map((region) => region.key));
+	const regions: NoteRegion[] = topLevels.map((region) => ({
+		key: region.key,
+		label: region.label,
+		status: "seen",
+	}));
+
+	if (prev) {
+		for (const old of prev.regions) {
+			if (old.status === "never-looked") continue;
+			if (!currentKeys.has(old.key)) {
+				regions.push({ ...old, status: "changed", detail: "not matched in latest look" });
+			}
+		}
+	}
+
+	if (!window.windowRef || window.pairingScore === Number.NEGATIVE_INFINITY) {
+		regions.push({
+			key: "window:unpaired",
+			label: "window capture",
+			status: "never-looked",
+			detail: "AX window without capture pairing",
+		});
+	}
+
+	regions.push(...frontierRegions(outline, topLevels));
+
+	return {
+		windowRef: window.windowRef ?? "(unpaired)",
+		title: window.title,
+		pairing: window.pairing ?? "low",
+		lastLookId: outline.lookId,
+		regions: uniqueRegions(regions),
+	};
+}
+
+export function noteAfterAct(prev: WindowNote | undefined, targetRef: string | undefined, outline: Outline, refreshOutcome: NoteRefreshOutcome): WindowNote {
+	const note = noteFromLook(prev, outline, refreshOutcome.window);
+	if (targetRef) {
+		const target = nodeByRef(outline, targetRef);
+		if (target) {
+			const ancestor = topLevelAncestor(target, outline);
+			const key = regionKey(ancestor);
+			const region = note.regions.find((candidate) => candidate.key === key);
+			if (region) {
+				region.status = "changed";
+				region.detail = "acted here";
+			} else {
+				note.regions.unshift({ key, label: regionLabel(ancestor), status: "changed", detail: "acted here" });
+			}
+		}
+	}
+	for (const delta of refreshOutcome.rootDelta ?? []) {
+		note.regions.unshift({
+			key: `root:${delta.change}:${delta.ref ?? delta.kind}:${delta.title ?? ""}`,
+			label: `${delta.kind}${delta.title ? ` ${delta.title}` : ""}`,
+			status: delta.change === "closed" ? "changed" : "never-looked",
+			detail: `root ${delta.change}`,
+		});
+	}
+	if (refreshOutcome.windowChanged) {
+		for (const region of note.regions) {
+			if (region.status === "seen") region.status = "changed";
+		}
+		note.regions.push({
+			key: `window:changed:${note.lastLookId ?? "unknown"}`,
+			label: refreshOutcome.newWindowLabel ?? "new sheet/window",
+			status: "never-looked",
+			detail: "appeared after act",
+		});
+	}
+	return { ...note, regions: uniqueRegions(note.regions) };
+}
+
+export function noteRegionKeyForRef(outline: Outline, ref: string): string | undefined {
+	const node = nodeByRef(outline, ref);
+	return node ? regionKey(topLevelAncestor(node, outline)) : undefined;
+}
+
+export function renderNote(note: WindowNote | undefined): string {
+	if (!note) return "";
+	const looked = note.lastLookId ? "looked just now" : "not looked";
+	const lines = [`note ${note.windowRef} ${JSON.stringify(note.title.slice(0, 512))} (pairing ${note.pairing}, ${looked})`];
+	const visible = note.regions.slice(0, 64);
+	for (const region of visible) {
+		const detail = region.detail ? `   (${region.detail.slice(0, 256)})` : "";
+		lines.push(`  ${region.label.slice(0, 512).padEnd(14, " ")} ${region.status}${detail}`);
+	}
+	if (note.regions.length > visible.length) lines.push(`  … ${note.regions.length - visible.length} more note regions; use search_ui or expand_ui`);
+	const encoded = new TextEncoder().encode(lines.join("\n"));
+	if (encoded.byteLength <= 8 * 1024) return new TextDecoder().decode(encoded);
+	let end = 8 * 1024;
+	while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1;
+	return `${new TextDecoder().decode(encoded.subarray(0, end))}\n… note byte budget reached`;
+}