@xopcai/xopc 0.0.285 → 0.0.291

package/dist/src/computer/broker.js

@@ -0,0 +1,337 @@
+import { ComputerConfigSchema, init_config } from "./config.js";
+import { ActionEnvelopeSchema, ComputerCommandSchema, init_src } from "../../packages/computer-control-contract/src/index.js";
+import { ComputerTargetError } from "./errors.js";
+import { createHash, randomUUID } from "node:crypto";
+//#region src/computer/broker.ts
+init_src();
+init_config();
+/** Main-process-only authority. Never expose this object or its approval hook to a renderer. */
+var ComputerBroker = class {
+	epoch = randomUUID();
+	session;
+	busy = false;
+	stopping;
+	discovery;
+	apps = /* @__PURE__ */ new Map();
+	config;
+	timer;
+	constructor(driver, host, config = {}, now = Date.now) {
+		this.driver = driver;
+		this.host = host;
+		this.now = now;
+		this.config = ComputerConfigSchema.parse(config);
+		this.timer = setInterval(() => {
+			const s = this.session;
+			if (s && s.status !== "stopped" && (this.expired(s) || !this.host.isVisible())) this.stop().catch(() => {});
+		}, 1e3);
+		this.timer.unref?.();
+	}
+	expired(s) {
+		return this.now() - s.createdAt >= this.config.maxSessionDurationMs || this.now() - s.touchedAt >= this.config.idleGrantTtlMs;
+	}
+	result(s) {
+		return {
+			sessionId: s.id,
+			brokerEpoch: this.epoch,
+			generation: s.generation,
+			status: s.status,
+			...s.errorCode ? { errorCode: s.errorCode } : {},
+			...s.windows ? { windows: s.windows } : {},
+			...s.status === "ready" ? { grantId: s.grantId } : {}
+		};
+	}
+	active(s) {
+		if (this.session !== s || s.controller.signal.aborted || this.expired(s) || !this.host.isVisible()) throw new Error("COMPUTER_GRANT_REVOKED");
+	}
+	async command(raw) {
+		const command = ComputerCommandSchema.parse(raw);
+		if (command.op === "discover") return this.discover(command);
+		if (command.op === "open") return this.open(command);
+		const s = this.session;
+		if (!s || s.id !== command.sessionId || s.owner !== command.owner) throw new Error("COMPUTER_SESSION_NOT_FOUND");
+		if (command.op === "release") {
+			await this.stop();
+			return this.result(s);
+		}
+		if (command.op === "status") return this.result(s);
+		this.active(s);
+		if (this.busy || s.status !== "ready" && s.status !== "pending_action") throw new Error("COMPUTER_NOT_READY");
+		this.busy = true;
+		try {
+			if (command.op === "observe") {
+				if (s.pending) throw new Error("COMPUTER_ACTION_PENDING");
+				return await this.observe(s);
+			}
+			return await this.act(s, command.envelope);
+		} finally {
+			this.busy = false;
+			this.host.onStateChange?.();
+		}
+	}
+	async discover(command) {
+		if (!this.config.enabled) throw new Error("COMPUTER_DISABLED");
+		if (!this.host.isVisible()) throw new Error("COMPUTER_LOCAL_UI_REQUIRED");
+		if (this.busy || this.discovery || this.stopping || this.session && this.session.status !== "stopped" && (this.session.owner !== command.owner || this.session.status !== "ready")) throw new Error("COMPUTER_BUSY");
+		const controller = new AbortController();
+		this.discovery = {
+			controller,
+			owner: command.owner,
+			id: command.sessionId
+		};
+		this.busy = true;
+		try {
+			const found = await this.driver.discover(command.query, controller.signal);
+			controller.signal.throwIfAborted();
+			if (!this.host.isVisible()) throw new Error("COMPUTER_LOCAL_UI_REQUIRED");
+			for (const [ref, app] of this.apps) if (app.owner === command.owner || app.expires <= this.now()) this.apps.delete(ref);
+			const apps = found.slice(0, 100).map((app) => {
+				const appRef = randomUUID();
+				this.apps.set(appRef, {
+					owner: command.owner,
+					appId: app.appId,
+					name: app.name,
+					expires: this.now() + 3e5
+				});
+				return {
+					appRef,
+					name: app.name,
+					running: app.running
+				};
+			});
+			while (this.apps.size > 500) this.apps.delete(this.apps.keys().next().value);
+			return {
+				status: "ready",
+				sessionId: command.sessionId,
+				brokerEpoch: this.epoch,
+				generation: 0,
+				apps
+			};
+		} finally {
+			try {
+				if (!this.session || this.session.status === "stopped") await this.driver.stop();
+			} finally {
+				this.discovery = void 0;
+				this.busy = false;
+			}
+		}
+	}
+	async open(command) {
+		if (!this.config.enabled) throw new Error("COMPUTER_DISABLED");
+		if (!this.host.isVisible()) throw new Error("COMPUTER_LOCAL_UI_REQUIRED");
+		if (this.stopping) throw new Error("COMPUTER_STOPPING");
+		if (this.discovery || this.busy) throw new Error("COMPUTER_BUSY");
+		if (this.session && this.session.status !== "stopped") {
+			const s = this.session;
+			if (s.id !== command.sessionId || s.owner !== command.owner || s.appRef !== command.appRef || s.mode !== command.mode || s.prepare !== command.prepare || s.windowRef !== command.windowRef || JSON.stringify(s.model) !== JSON.stringify(command.model)) throw new Error("COMPUTER_DEVICE_BUSY");
+			return this.result(s);
+		}
+		const app = this.apps.get(command.appRef);
+		if (!app || app.owner !== command.owner || app.expires <= this.now()) throw new Error("COMPUTER_APP_REF_EXPIRED");
+		const s = {
+			id: command.sessionId,
+			owner: command.owner,
+			appId: app.appId,
+			model: command.model,
+			appRef: command.appRef,
+			mode: command.mode,
+			prepare: command.prepare,
+			windowRef: command.windowRef,
+			grantId: randomUUID(),
+			generation: 0,
+			status: "pending_authorization",
+			createdAt: this.now(),
+			touchedAt: this.now(),
+			controller: new AbortController(),
+			receipts: /* @__PURE__ */ new Map(),
+			actions: 0
+		};
+		this.session = s;
+		const fullControl = this.host.hasFullControl?.() === true;
+		const authorization = (fullControl ? Promise.resolve(true) : this.host.requestApproval({
+			kind: "session",
+			id: s.id,
+			appId: s.appId,
+			appName: app.name,
+			mode: s.mode,
+			prepare: s.prepare,
+			model: s.model
+		}, s.controller.signal)).then(async (approved) => {
+			this.active(s);
+			if (!approved) {
+				await this.stop();
+				return;
+			}
+			s.target = await this.driver.resolveTarget(s.appId, s.controller.signal, {
+				prepare: s.prepare,
+				windowRef: s.windowRef
+			});
+			this.active(s);
+			if (s.target.appId !== s.appId) throw new Error("COMPUTER_TARGET_MISMATCH");
+			s.status = "ready";
+			s.touchedAt = this.now();
+		}).catch((error) => {
+			if (this.session !== s) return;
+			s.errorCode = error instanceof Error && /^COMPUTER_[A-Z_]+$/.test(error.message) ? error.message : "COMPUTER_NATIVE_SETUP_FAILED";
+			if (error instanceof ComputerTargetError) s.windows = error.windows;
+			return this.stop().catch(() => {});
+		}).finally(() => this.host.onStateChange?.());
+		if (fullControl) await authorization;
+		return this.result(s);
+	}
+	async observe(s) {
+		const frame = await this.driver.observe(s.target, s.controller.signal);
+		try {
+			this.active(s);
+			if (frame.target.appId !== s.appId || frame.target.processIdentity !== s.target.processIdentity || frame.target.windowId !== s.target.windowId) throw new Error("COMPUTER_TARGET_CHANGED");
+			if (frame.image.byteLength > 5 * 1024 * 1024 || frame.imageWidth * frame.imageHeight > 16e6) throw new Error("COMPUTER_FRAME_LIMIT");
+			s.target = frame.target;
+			s.observation = {
+				id: randomUUID(),
+				sessionId: s.id,
+				brokerEpoch: this.epoch,
+				generation: s.generation,
+				target: frame.target,
+				capturedAt: this.now(),
+				imageWidth: frame.imageWidth,
+				imageHeight: frame.imageHeight,
+				summary: frame.summary.slice(0, 12e3),
+				focusedEditableRef: frame.focusedEditableRef,
+				stateDigest: frame.stateDigest
+			};
+			s.touchedAt = this.now();
+			return {
+				...this.result(s),
+				observation: s.observation,
+				frame: {
+					bytes: frame.image,
+					mimeType: frame.mimeType
+				}
+			};
+		} catch (error) {
+			frame.image.fill(0);
+			throw error;
+		}
+	}
+	async act(s, raw) {
+		if (s.mode === "observe") throw new Error("COMPUTER_READ_ONLY");
+		const e = ActionEnvelopeSchema.parse(raw);
+		if (e.sessionId !== s.id || e.brokerEpoch !== this.epoch || e.generation !== s.generation || e.grantId !== s.grantId || e.deadlineAt <= this.now()) throw new Error("COMPUTER_STALE_ACTION");
+		const digest = createHash("sha256").update(JSON.stringify(e)).digest("hex");
+		const previous = s.receipts.get(e.actionId);
+		if (previous) {
+			if (previous.digest !== digest) throw new Error("COMPUTER_ACTION_ID_REUSED");
+			return {
+				...this.result(s),
+				receipt: previous.receipt
+			};
+		}
+		const observed = s.observation;
+		if (!observed || e.observationId !== observed.id || this.now() - observed.capturedAt > 12e4) throw new Error("COMPUTER_STALE_OBSERVATION");
+		if (s.actions >= this.config.maxActionsPerSession) throw new Error("COMPUTER_ACTION_BUDGET");
+		if ("point" in e.action && e.action.point && (e.action.point.x >= observed.imageWidth || e.action.point.y >= observed.imageHeight)) throw new Error("COMPUTER_COORDINATE_OUTSIDE_WINDOW");
+		if (e.action.kind === "typeText" && !e.action.point && !observed.focusedEditableRef) throw new Error("COMPUTER_EDITABLE_TARGET_REQUIRED");
+		this.driver.validateAction?.(e.action);
+		if (e.action.kind !== "wait" && !this.host.hasFullControl?.()) {
+			if (!s.pending) {
+				s.pending = {
+					envelope: e,
+					digest
+				};
+				s.status = "pending_action";
+				this.host.requestApproval({
+					kind: "action",
+					id: e.actionId,
+					target: observed.target,
+					action: e.action
+				}, s.controller.signal).then((approved) => {
+					this.active(s);
+					if (s.pending?.digest === digest) s.pending.approved = approved;
+				}).catch(() => {}).finally(() => this.host.onStateChange?.());
+				return this.result(s);
+			}
+			if (s.pending.digest !== digest) throw new Error("COMPUTER_DIFFERENT_ACTION_PENDING");
+			if (s.pending.approved === void 0) return this.result(s);
+			if (!s.pending.approved) {
+				await this.stop();
+				return this.result(s);
+			}
+		}
+		const current = await this.driver.observe(s.target, s.controller.signal);
+		try {
+			this.active(s);
+			if (JSON.stringify(current.target) !== JSON.stringify(observed.target) || current.stateDigest !== observed.stateDigest || e.deadlineAt <= this.now()) {
+				s.pending = void 0;
+				s.observation = void 0;
+				s.status = "ready";
+				throw new Error("COMPUTER_APPROVAL_TARGET_CHANGED");
+			}
+			this.driver.validateAction?.(e.action);
+		} finally {
+			current.image.fill(0);
+		}
+		const receipt = {
+			actionId: e.actionId,
+			dispatch: "started",
+			outcome: "unknown",
+			verification: "none"
+		};
+		s.receipts.set(e.actionId, {
+			digest,
+			receipt
+		});
+		s.actions++;
+		s.pending = void 0;
+		s.status = "running";
+		try {
+			await this.driver.perform(s.target, e.action, s.controller.signal);
+			this.active(s);
+			receipt.dispatch = "completed";
+			s.status = "ready";
+			const after = await this.observe(s);
+			receipt.verification = after.observation ? "visual" : "none";
+			return {
+				...after,
+				receipt
+			};
+		} catch {
+			receipt.dispatch = "unknown";
+			await this.stop();
+			return {
+				...this.result(s),
+				receipt
+			};
+		}
+	}
+	stop() {
+		this.discovery?.controller.abort();
+		const s = this.session;
+		if (s && s.status !== "stopped") {
+			s.generation++;
+			s.status = "stopped";
+			s.pending = void 0;
+			s.observation = void 0;
+			s.controller.abort();
+		}
+		if (!this.stopping) this.stopping = this.driver.stop().finally(() => {
+			this.stopping = void 0;
+			this.host.onStateChange?.();
+		});
+		return this.stopping;
+	}
+	async cancel(command) {
+		if (this.session?.id === command.sessionId && this.session.owner === command.owner || this.discovery?.id === command.sessionId && this.discovery.owner === command.owner) await this.stop();
+	}
+	snapshot() {
+		return this.session ? {
+			status: this.session.status,
+			errorCode: this.session.errorCode,
+			appId: this.session.appId
+		} : { status: "idle" };
+	}
+	async dispose() {
+		clearInterval(this.timer);
+		await this.stop();
+	}
+};
+//#endregion
+export { ComputerBroker };

package/dist/src/computer/config.d.ts

@@ -0,0 +1,9 @@
+import { z } from 'zod';
+export declare const ComputerConfigSchema: z.ZodPrefault<z.ZodObject<{
+    enabled: z.ZodDefault<z.ZodBoolean>;
+    maxActionsPerSession: z.ZodDefault<z.ZodNumber>;
+    maxModelRequests: z.ZodDefault<z.ZodNumber>;
+    maxSessionDurationMs: z.ZodDefault<z.ZodNumber>;
+    idleGrantTtlMs: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strict>>;
+export type ComputerConfig = z.infer<typeof ComputerConfigSchema>;

package/dist/src/computer/config.js

@@ -0,0 +1,16 @@
+import { __esmMin } from "../../_virtual/_rolldown/runtime.js";
+import { z } from "zod";
+//#region src/computer/config.ts
+var ComputerConfigSchema;
+var init_config = __esmMin((() => {
+	ComputerConfigSchema = z.object({
+		enabled: z.boolean().default(false),
+		maxActionsPerSession: z.number().int().min(1).max(80).default(80),
+		maxModelRequests: z.number().int().min(1).max(120).default(120),
+		maxSessionDurationMs: z.number().int().min(1e3).max(9e5).default(9e5),
+		idleGrantTtlMs: z.number().int().min(1e3).max(3e5).default(3e5)
+	}).strict().prefault({});
+}));
+//#endregion
+init_config();
+export { ComputerConfigSchema, init_config };

package/dist/src/computer/control-guard.d.ts

@@ -0,0 +1,4 @@
+export declare function enterComputerControl(owner: string, close: () => Promise<void>): void;
+export declare function leaveComputerControl(owner: string): void;
+export declare function isComputerControlActive(owner: string): boolean;
+export declare function stopComputerControl(owner: string): Promise<void>;

package/dist/src/computer/control-guard.js

@@ -0,0 +1,17 @@
+//#region src/computer/control-guard.ts
+/** Process-local admission gate; authority itself remains in the desktop broker. */
+const owners = /* @__PURE__ */ new Map();
+function enterComputerControl(owner, close) {
+	owners.set(owner, close);
+}
+function leaveComputerControl(owner) {
+	owners.delete(owner);
+}
+function isComputerControlActive(owner) {
+	return owners.has(owner);
+}
+async function stopComputerControl(owner) {
+	await owners.get(owner)?.();
+}
+//#endregion
+export { enterComputerControl, isComputerControlActive, leaveComputerControl, stopComputerControl };

package/dist/src/computer/errors.d.ts

@@ -0,0 +1,7 @@
+import type { ComputerWindow } from '@xopcai/computer-control-contract';
+export declare class ComputerTargetError extends Error {
+    readonly windows: ComputerWindow[];
+    constructor(code: string, windows?: ComputerWindow[]);
+}
+/** Recovery instructions are deterministic; never replay an input with unknown delivery. */
+export declare function computerRecovery(code: string): string;

package/dist/src/computer/errors.js

@@ -0,0 +1,30 @@
+//#region src/computer/errors.ts
+var ComputerTargetError = class extends Error {
+	constructor(code, windows = []) {
+		super(code);
+		this.windows = windows;
+	}
+};
+/** Recovery instructions are deterministic; never replay an input with unknown delivery. */
+function computerRecovery(code) {
+	switch (code) {
+		case "COMPUTER_INVALID_INPUT": return "Read tool_manual(computer_use) and use only the fields documented for this operation. Start with discover; open requires appRef, mode and prepare.";
+		case "COMPUTER_APP_INSTANCE_AMBIGUOUS": return "Multiple processes for this app are running. Ask the user to keep the intended app instance open; do not guess a process.";
+		case "COMPUTER_READ_ONLY_MODEL_OUTPUT": return "The GUI model did not provide a read-only answer. No input was executed. Report the limitation; do not retry as an action or change providers.";
+		case "COMPUTER_APP_REF_EXPIRED": return "Discover the named app again, then use the returned appRef.";
+		case "COMPUTER_APP_NOT_FOUND": return "Discover again. If absent, ask the user to install the app; do not guess an identifier.";
+		case "COMPUTER_APP_NOT_RUNNING": return "Open with prepare=true only if the user requested launching the app; otherwise ask them to open it.";
+		case "COMPUTER_WINDOW_REQUIRED": return "No visible target window. If the task permits bringing the app forward, open with prepare=true; otherwise ask the user to show the window. If preparation already failed, require a local state change before retrying.";
+		case "COMPUTER_WINDOW_AMBIGUOUS": return "Choose a returned windowRef only when its title matches the task. Otherwise ask the user using window titles, not IDs.";
+		case "COMPUTER_WINDOW_CHANGED": return "The selected window no longer exists. Discover and open again; never reuse its coordinates.";
+		case "COMPUTER_READ_ONLY": return "This session is read-only. Do not retry an action or reopen in control mode without user authorization.";
+		case "COMPUTER_DEVICE_BUSY":
+		case "COMPUTER_BUSY": return "Another desktop operation is active. Wait for it to finish; do not stop another task.";
+		case "COMPUTER_CONTROL_PAUSED":
+		case "COMPUTER_LOCAL_UI_REQUIRED": return "Ask the user to resume desktop control locally. Do not automatically resume or bypass this stop.";
+		case "COMPUTER_OS_PERMISSION_REQUIRED": return "Ask the user to check Accessibility and Screen Recording in macOS settings.";
+		default: return "Do not repeat the same call without a state change. Report this error; never bypass desktop restrictions with shell or other tools.";
+	}
+}
+//#endregion
+export { ComputerTargetError, computerRecovery };

package/dist/src/computer/model-adapter.d.ts

@@ -0,0 +1,44 @@
+import { type ComputerAction, type ComputerProfile } from '@xopcai/computer-control-contract';
+export type ComputerProposal = {
+    kind: 'action';
+    action: ComputerAction;
+} | {
+    kind: 'finished';
+    claimedSuccess: boolean;
+} | {
+    kind: 'answer';
+    text: string;
+} | {
+    kind: 'takeover';
+    reason: string;
+};
+export declare const GUI_PLUS_SYSTEM_PROMPT: string;
+export declare function parseGuiPlusProposal(content: string, width: number, height: number): ComputerProposal;
+export interface ComputerModelConnection {
+    modelId: string;
+    baseUrl: string;
+    apiKey: string;
+    profile: ComputerProfile;
+    deploymentRevision?: string;
+    headers?: Record<string, string>;
+}
+export interface ComputerPredictionInput {
+    goal: string;
+    image: Uint8Array;
+    mimeType: 'image/png' | 'image/jpeg';
+    width: number;
+    height: number;
+    summary: string;
+    formatCorrection?: boolean;
+    readOnly?: boolean;
+}
+/** One format-only re-prediction is safe before any input has been proposed or dispatched. */
+export declare function predictComputerStep(adapter: ComputerModelAdapter, input: ComputerPredictionInput, reserveAttempt: () => void, signal?: AbortSignal): Promise<ComputerProposal>;
+export declare function readComputerJson(response: Response, maxBytes: number): Promise<unknown>;
+/** A frozen single-recipient connection; never retries, redirects or changes billing sources. */
+export declare class ComputerModelAdapter {
+    private readonly fetchImpl;
+    private readonly connection;
+    constructor(connection: ComputerModelConnection, fetchImpl?: typeof fetch);
+    predict(input: ComputerPredictionInput, signal?: AbortSignal): Promise<ComputerProposal>;
+}