@xopcai/xopc 0.0.391 → 0.0.393

package/dist/src/computer/runtime.js

@@ -8,6 +8,9 @@ import { rejectUnbudgetedCloudTransport } from "../providers/xopc-cloud-credit-b
 import { enterComputerControl, leaveComputerControl } from "./control-guard.js";
 import { ComputerDiagnosticSchema, ComputerOperationError, computerDiagnostic, computerRecovery } from "./errors.js";
 import { createComputerModelAdapter, predictComputerStep, readComputerJson } from "./model-adapter.js";
+import { groundComputerAction } from "./semantic-actions.js";
+import { ComputerFlowCapture, ComputerFlowStore, computerFlowHint, computerFlowMetadata } from "./flows.js";
+import { ComputerSessionDiagnostics } from "./diagnostics.js";
 import { ComputerExpectationSchema, ComputerTaskState, verifyComputerExpectation } from "./task-state.js";
 import { randomUUID } from "node:crypto";
 import { z } from "zod";
@@ -77,24 +80,58 @@ const ComputerUseInputSchema = z.discriminatedUnion("op", [
 	z.object({
 		op: z.literal("step"),
 		goal: z.string().min(1).max(4e3),
-		expect: ComputerExpectationSchema.optional()
+		expect: ComputerExpectationSchema.optional(),
+		flowId: z.uuid().optional()
+	}).strict(),
+	z.object({
+		op: z.literal("close"),
+		saveFlowAs: z.string().trim().min(1).max(160).optional()
+	}).strict(),
+	z.object({ op: z.literal("flows") }).strict(),
+	z.object({
+		op: z.literal("forget_flow"),
+		flowId: z.uuid()
 	}).strict(),
-	z.object({ op: z.literal("close") }).strict()
+	z.object({ op: z.literal("report") }).strict()
 ]);
 var ComputerRuntime = class {
 	sessions = /* @__PURE__ */ new Map();
 	operations = /* @__PURE__ */ new Map();
-	constructor(endpoints, getConfig, getTurnEndpoint) {
+	reports = /* @__PURE__ */ new Map();
+	openingReports = /* @__PURE__ */ new Map();
+	constructor(endpoints, getConfig, getTurnEndpoint, flowStore = new ComputerFlowStore()) {
 		this.endpoints = endpoints;
 		this.getConfig = getConfig;
 		this.getTurnEndpoint = getTurnEndpoint;
+		this.flowStore = flowStore;
 	}
 	async execute(owner, input, signal) {
 		input = ComputerUseInputSchema.parse(input);
 		if (!owner) throw new Error("COMPUTER_CONVERSATION_REQUIRED");
+		if (input.op === "flows" || input.op === "forget_flow") {
+			signal?.throwIfAborted();
+			const scope = this.flowScope(owner);
+			return input.op === "flows" ? {
+				status: "flows",
+				sessionId: "none",
+				flows: this.flowStore.list(scope).map(computerFlowMetadata)
+			} : {
+				status: this.flowStore.delete(scope, input.flowId) ? "flow_deleted" : "flow_not_found",
+				sessionId: "none"
+			};
+		}
+		if (input.op === "report") {
+			const report = this.getReport(owner);
+			return {
+				status: report ? "report" : "no_report",
+				sessionId: report?.sessionId ?? "none",
+				...report ? { report } : {}
+			};
+		}
 		if (input.op === "close") {
 			const session = this.sessions.get(owner);
 			const sessionId = session?.id ?? "none";
+			const canSaveFlow = session && !session.pending && !this.operations.has(owner) && !signal?.aborted && session.mode === "control";
 			const handoff = { recentActions: (session?.task.history ?? []).map(({ goal, action, dispatch, outcome, verification }) => ({
 				goal,
 				action,
@@ -102,28 +139,101 @@ var ComputerRuntime = class {
 				outcome,
 				verification
 			})) };
-			return await this.close(owner) ? {
+			const released = await this.close(owner);
+			const report = this.getReport(owner);
+			let savedFlow;
+			if (input.saveFlowAs && released) {
+				if (!canSaveFlow || !report || report.failure || report.counters.unknownDispatches || report.counters.takeovers) return {
+					status: "stopped",
+					sessionId,
+					errorCode: "COMPUTER_FLOW_NOT_VERIFIED",
+					report,
+					nextAction: "Control was released. Save only a complete native navigation sequence with a final verified selected condition or positive page-specific text and no failure or takeover."
+				};
+				let draft;
+				try {
+					draft = session.flowCapture.finish(input.saveFlowAs, session.endpointId);
+				} catch {
+					return {
+						status: "stopped",
+						sessionId,
+						errorCode: "COMPUTER_FLOW_NOT_VERIFIED",
+						report,
+						nextAction: "Control was released. Only a complete native navigation sequence with non-sensitive labels and a final verified selected condition or positive page-specific text can be saved."
+					};
+				}
+				try {
+					savedFlow = computerFlowMetadata(this.flowStore.save(session.flowScope, {
+						...draft,
+						...session.flowPrincipalId ? { principalId: session.flowPrincipalId } : {}
+					}));
+				} catch {
+					return {
+						status: "stopped",
+						sessionId,
+						errorCode: "COMPUTER_FLOW_SAVE_FAILED",
+						report,
+						nextAction: "Control was released; no reusable flow was confirmed saved. Check native verification, non-sensitive labels and local storage. Do not repeat desktop input to save a flow."
+					};
+				}
+			}
+			return released ? {
 				status: "stopped",
 				sessionId,
 				handoff,
+				...savedFlow ? { savedFlow } : {},
+				...report ? { report } : {},
 				nextAction: "Desktop control is released. Continue with an appropriate authorized tool; this does not permit bypassing a desktop refusal."
 			} : {
 				status: "stop_unconfirmed",
 				sessionId,
 				errorCode: "COMPUTER_RELEASE_UNCONFIRMED",
+				report,
 				nextAction: computerRecovery("COMPUTER_RELEASE_UNCONFIRMED")
 			};
 		}
 		if (this.operations.has(owner)) throw new Error("COMPUTER_BUSY");
 		const operation = new AbortController();
 		this.operations.set(owner, operation);
+		const diagnostics = this.sessions.get(owner)?.diagnostics ?? (input.op === "open" ? new ComputerSessionDiagnostics() : void 0);
+		if (diagnostics) this.openingReports.set(owner, diagnostics);
 		try {
-			return await this.executeInput(owner, input, signal ? AbortSignal.any([signal, operation.signal]) : operation.signal);
+			const result = await this.executeInput(owner, input, signal ? AbortSignal.any([signal, operation.signal]) : operation.signal, diagnostics);
+			if (result.errorCode) diagnostics?.fail(new Error(result.errorCode), diagnostics.phase, result.receipt?.dispatch ?? "notStarted");
+			if (diagnostics && !this.sessions.has(owner)) this.retainReport(owner, diagnostics);
+			return {
+				...result,
+				...diagnostics ? {
+					reportId: diagnostics.snapshot().reportId,
+					...result.errorCode || result.receipt?.dispatch === "unknown" ? { report: diagnostics.snapshot() } : {}
+				} : {}
+			};
+		} catch (error) {
+			diagnostics?.fail(error, diagnostics.phase, void 0, !!signal?.aborted || operation.signal.aborted);
+			if (diagnostics && !this.sessions.has(owner)) {
+				diagnostics.finish(true);
+				this.retainReport(owner, diagnostics);
+			}
+			if (diagnostics) {
+				const report = diagnostics.snapshot();
+				log.warn({
+					reportId: report.reportId,
+					conversationId: owner,
+					sessionId: report.sessionId,
+					...report.failure
+				}, `Computer operation failed: ${report.failure?.errorCode}`);
+			}
+			throw error;
 		} finally {
 			if (this.operations.get(owner) === operation) this.operations.delete(owner);
+			if (this.openingReports.get(owner) === diagnostics) this.openingReports.delete(owner);
 		}
 	}
-	async executeInput(owner, input, signal) {
+	flowScope(owner) {
+		const effective = resolveEffectiveAgentConfigForSession(owner).config;
+		return JSON.stringify([effective.id, effective.workspace]);
+	}
+	async executeInput(owner, input, signal, diagnostics) {
 		signal.throwIfAborted();
 		const config = this.getConfig();
 		if (!config.computer.enabled) {
@@ -186,6 +296,7 @@ var ComputerRuntime = class {
 			const adapter = createComputerModelAdapter({
 				modelId: model.id,
 				baseUrl: model.baseUrl,
+				api: model.api,
 				apiKey,
 				profile,
 				headers: model.headers,
@@ -212,10 +323,17 @@ var ComputerRuntime = class {
 				createdAt: Date.now(),
 				actions: 0,
 				modelRequests: 0,
+				rescues: 0,
+				rescueLimit: config.computer.maxRescuesPerSession,
 				controller: new AbortController(),
 				task: new ComputerTaskState(),
+				flowCapture: new ComputerFlowCapture(),
+				flowPrincipalId: endpoint.principalId,
+				flowScope: JSON.stringify([effective.id, effective.workspace]),
+				diagnostics,
 				serviceLimits
 			};
+			s.diagnostics.bind(s.id);
 			this.sessions.set(owner, s);
 			enterComputerControl(owner, async () => {
 				await this.close(owner);
@@ -224,10 +342,12 @@ var ComputerRuntime = class {
 		if (!s) throw new Error("COMPUTER_OPEN_REQUIRED");
 		signal = signal ? AbortSignal.any([signal, s.controller.signal]) : s.controller.signal;
 		if (Date.now() - s.createdAt >= config.computer.maxSessionDurationMs) {
+			s.diagnostics.fail(/* @__PURE__ */ new Error("COMPUTER_SESSION_BUDGET"), "observe", "notStarted");
 			await this.close(owner);
 			throw new Error("COMPUTER_SESSION_BUDGET");
 		}
 		let phase = "observe";
+		s.diagnostics.phase = "observe";
 		try {
 			if (input.op === "open") {
 				if (input.appRef !== s.appRef || input.windowRef !== s.windowRef || input.mode !== s.mode || input.prepare !== s.prepare) throw new Error("COMPUTER_CLOSE_BEFORE_TARGET_CHANGE");
@@ -241,6 +361,7 @@ var ComputerRuntime = class {
 					...s.windowRef ? { windowRef: s.windowRef } : {},
 					model: s.binding
 				}, signal);
+				if (reply.value.errorCode) s.diagnostics.fail(new Error(reply.value.errorCode), "open", "notStarted");
 				if (reply.value.status === "stopped") this.forget(s);
 				return {
 					...this.present(reply.value),
@@ -253,17 +374,33 @@ var ComputerRuntime = class {
 				owner
 			}, signal);
 			if (status.value.status === "stopped") {
+				if (status.value.errorCode) s.diagnostics.fail(new Error(status.value.errorCode), "observe", "notStarted");
 				this.forget(s);
 				return this.present(status.value);
 			}
 			if (input.op === "step" && s.mode === "observe") throw new Error("COMPUTER_READ_ONLY");
 			if (s.pending) {
 				if (input.op === "observe") return this.present(status.value);
+				const context = s.pendingContext;
 				phase = "dispatch";
-				return await this.dispatchAction(s, s.pending, signal);
+				const result = await this.dispatchAction(s, s.pending, signal);
+				if (!this.isUndispatchedChange(result) || !context) return result;
+				input = {
+					op: "step",
+					goal: context.goal,
+					expect: context.expectation,
+					flowId: context.flowId
+				};
+				this.reserveRescue(s, signal);
+			} else if (status.value.status !== "ready") return this.present(status.value);
+			if (input.op === "step") {
+				if (input.flowId || s.lastFlowId) s.adapter.reset?.();
+				s.lastFlowId = input.flowId;
 			}
-			if (status.value.status !== "ready") return this.present(status.value);
-			for (let observationAttempt = 0; observationAttempt < 2; observationAttempt++) {
+			while (true) {
+				signal.throwIfAborted();
+				if (Date.now() - s.createdAt >= config.computer.maxSessionDurationMs) throw new Error("COMPUTER_SESSION_BUDGET");
+				phase = "observe";
 				const observed = await this.send(s, {
 					op: "observe",
 					sessionId: s.id,
@@ -271,6 +408,7 @@ var ComputerRuntime = class {
 				}, signal);
 				try {
 					const verification = verifyComputerExpectation(input.expect, observed.value.observation);
+					if (input.op === "observe") s.diagnostics.verify(verification);
 					if (input.op === "observe" && !input.question) return {
 						...this.present(observed.value),
 						verification,
@@ -278,6 +416,7 @@ var ComputerRuntime = class {
 						budget: this.budget(s)
 					};
 					if (input.op === "step" && input.expect?.kind !== "text" && verification?.status === "satisfied") {
+						s.diagnostics.verify(verification);
 						s.adapter.reset?.();
 						return {
 							status: "condition_satisfied",
@@ -291,10 +430,19 @@ var ComputerRuntime = class {
 					if (input.op === "step" && s.actions >= config.computer.maxActionsPerSession) throw new Error("COMPUTER_ACTION_BUDGET");
 					const o = observed.value.observation;
 					if (!o || !observed.frame || !observed.mimeType) throw new Error("COMPUTER_FRAME_REQUIRED");
+					let flowHint;
+					if (input.op === "step" && input.flowId) {
+						const flowId = input.flowId;
+						const flow = this.flowStore.list(s.flowScope).find((item) => item.id === flowId);
+						if (!flow) throw new Error("COMPUTER_FLOW_NOT_FOUND");
+						if ((flow.principalId ? flow.principalId !== s.flowPrincipalId : flow.endpointId !== s.endpointId) || flow.appId !== o.target.appId) throw new Error("COMPUTER_FLOW_TARGET_MISMATCH");
+						flowHint = computerFlowHint(flow);
+					}
 					phase = "model";
-					const proposal = await predictComputerStep(s.adapter, {
-						goal: input.op === "observe" ? input.question : input.goal,
-						readOnly: input.op === "observe",
+					const predictionInput = input;
+					const proposal = await s.diagnostics.measure("model", () => predictComputerStep(s.adapter, {
+						goal: predictionInput.op === "observe" ? predictionInput.question : predictionInput.goal,
+						readOnly: predictionInput.op === "observe",
 						image: observed.frame,
 						mimeType: observed.mimeType,
 						width: o.imageWidth,
@@ -302,10 +450,13 @@ var ComputerRuntime = class {
 						summary: o.summary,
 						stateDigest: o.stateDigest,
 						history: s.task.history,
-						expectation: input.expect
+						expectation: predictionInput.expect,
+						flowHint,
+						onModelRequest: () => s.diagnostics.count("modelRequests")
 					}, () => {
 						if (++s.modelRequests > config.computer.maxModelRequests) throw new Error("COMPUTER_MODEL_BUDGET");
-					}, signal);
+						s.diagnostics.count("predictionAttempts");
+					}, signal));
 					if (proposal.kind === "answer") return {
 						...this.present(observed.value),
 						summary: proposal.text,
@@ -314,22 +465,29 @@ var ComputerRuntime = class {
 						budget: this.budget(s)
 					};
 					if (input.op === "observe") throw new Error("COMPUTER_READ_ONLY_MODEL_OUTPUT");
-					if (proposal.kind === "finished") return {
-						status: "model_finished",
-						sessionId: s.id,
-						claimedSuccess: proposal.claimedSuccess,
-						verified: false,
-						verification,
-						budget: this.budget(s),
-						nextAction: "The model claim is not proof. Check the requested completion condition using native or artifact evidence before reporting success."
-					};
-					if (proposal.kind === "takeover") return {
-						status: "takeover",
-						sessionId: s.id,
-						pending: true,
-						budget: this.budget(s),
-						summary: "The GUI model requests user assistance. Inspect the authorized window locally."
-					};
+					if (proposal.kind === "finished") {
+						if (proposal.claimedSuccess) s.diagnostics.count("successClaims");
+						return {
+							status: "model_finished",
+							sessionId: s.id,
+							claimedSuccess: proposal.claimedSuccess,
+							verified: false,
+							verification,
+							budget: this.budget(s),
+							nextAction: "The model claim is not proof. Check the requested completion condition using native or artifact evidence before reporting success."
+						};
+					}
+					if (proposal.kind === "takeover") {
+						s.diagnostics.count("takeovers");
+						return {
+							status: "takeover",
+							sessionId: s.id,
+							pending: true,
+							budget: this.budget(s),
+							summary: "The GUI model requests user assistance. Inspect the authorized window locally."
+						};
+					}
+					const action = groundComputerAction(proposal.action, o);
 					const e = {
 						actionId: randomUUID(),
 						sessionId: s.id,
@@ -338,44 +496,55 @@ var ComputerRuntime = class {
 						generation: o.generation,
 						grantId: observed.value.grantId,
 						deadlineAt: Date.now() + 12e4,
-						action: proposal.action
+						action
 					};
-					s.task.assertProgress(proposal.action, o);
+					s.diagnostics.phase = "dispatch";
+					s.task.assertProgress(action, o);
 					s.pendingContext = {
 						goal: input.goal,
 						before: o,
-						expectation: input.expect
+						expectation: input.expect,
+						flowId: input.flowId
 					};
 					phase = "dispatch";
 					const result = await this.dispatchAction(s, e, signal);
-					if (result.status !== "ready" || result.receipt || result.errorCode !== "COMPUTER_OBSERVATION_CHANGED") return result;
-					phase = "observe";
+					if (!this.isUndispatchedChange(result)) return result;
+					this.reserveRescue(s, signal);
 				} finally {
 					observed.frame?.fill(0);
 				}
 			}
-			throw new Error("COMPUTER_UI_UNSTABLE");
 		} catch (error) {
 			const failure = signal.aborted || computerDiagnostic(error) || error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error : new ComputerOperationError({
 				errorCode: `COMPUTER_${phase.toUpperCase()}_FAILED`,
 				phase,
 				diagnosticId: randomUUID()
 			});
-			const diagnostic = computerDiagnostic(failure);
-			if (diagnostic) log.warn({
-				...diagnostic,
-				conversationId: owner,
-				sessionId: s.id
-			}, `Computer operation failed: ${diagnostic.errorCode}`);
+			s.diagnostics.fail(failure, s.diagnostics.phase, void 0, signal.aborted);
 			await this.close(owner);
 			throw failure;
 		}
 	}
+	isUndispatchedChange(result) {
+		return result.status === "ready" && result.errorCode === "COMPUTER_OBSERVATION_CHANGED" && !result.receipt;
+	}
+	reserveRescue(s, signal) {
+		signal.throwIfAborted();
+		const limit = Math.min(s.rescueLimit, this.getConfig().computer.maxRescuesPerSession);
+		if (s.rescues >= limit) {
+			s.diagnostics.count("rescueExhaustions");
+			throw new Error("COMPUTER_UI_UNSTABLE");
+		}
+		s.rescues++;
+		s.diagnostics.count("rescueAttempts");
+		s.adapter.reset?.();
+	}
 	async dispatchAction(s, envelope, signal) {
 		if (!s.pending) {
 			if (s.actions >= this.getConfig().computer.maxActionsPerSession) throw new Error("COMPUTER_ACTION_BUDGET");
 			s.actions++;
 			s.pending = envelope;
+			s.diagnostics.propose(envelope.actionId, envelope.action.kind);
 		}
 		const reply = await this.send(s, {
 			op: "act",
@@ -383,9 +552,14 @@ var ComputerRuntime = class {
 			owner: s.owner,
 			envelope
 		}, signal);
+		if (reply.value.receipt && reply.value.receipt.actionId !== envelope.actionId) {
+			reply.frame?.fill(0);
+			throw new Error("COMPUTER_RECEIPT_MISMATCH");
+		}
 		if (reply.value.status !== "pending_action") s.pending = void 0;
 		reply.frame?.fill(0);
 		if (reply.value.status === "ready" && reply.value.errorCode === "COMPUTER_OBSERVATION_CHANGED" && !reply.value.receipt) {
+			s.diagnostics.discard();
 			s.actions--;
 			s.pendingContext = void 0;
 			return {
@@ -395,7 +569,13 @@ var ComputerRuntime = class {
 		}
 		const context = s.pendingContext;
 		const verification = verifyComputerExpectation(context?.expectation, reply.value.observation, context?.before);
-		if (context && reply.value.receipt) s.task.record(context.goal, envelope.action, context.before, reply.value.receipt, reply.value.observation, verification);
+		if (reply.value.receipt) s.diagnostics.record(reply.value.receipt, context?.before, reply.value.observation, verification);
+		const failureCode = reply.value.errorCode ?? (reply.value.receipt?.dispatch === "unknown" || reply.value.receipt?.dispatch === "started" ? reply.value.receipt.errorCode ?? "COMPUTER_NATIVE_ACTION_FAILED" : void 0);
+		if (failureCode) s.diagnostics.fail(new Error(failureCode), "dispatch", reply.value.receipt?.dispatch ?? "notStarted");
+		if (context && reply.value.receipt) {
+			s.task.record(context.goal, envelope.action, context.before, reply.value.receipt, reply.value.observation, verification);
+			s.flowCapture.record(envelope.action, context.before, reply.value.receipt, reply.value.observation, context.expectation, verification);
+		}
 		if (!s.pending) s.pendingContext = void 0;
 		if (reply.value.status === "stopped") this.forget(s);
 		return {
@@ -411,6 +591,7 @@ var ComputerRuntime = class {
 		return {
 			actionsRemaining: Math.max(0, config.maxActionsPerSession - s.actions),
 			modelRequestsRemaining: Math.max(0, config.maxModelRequests - s.modelRequests),
+			rescuesRemaining: Math.max(0, Math.min(s.rescueLimit, config.maxRescuesPerSession) - s.rescues),
 			expiresAt: s.createdAt + config.maxSessionDurationMs,
 			...s.serviceLimits ? { serviceLimits: s.serviceLimits } : {}
 		};
@@ -458,7 +639,7 @@ var ComputerRuntime = class {
 		signal?.throwIfAborted();
 		const tool = this.endpoints.registry.getTool(s.endpointId, COMPUTER_CONTROL_TOOL);
 		if (!tool) throw new Error("COMPUTER_DESKTOP_UPGRADE_REQUIRED");
-		const reply = await this.endpoints.invocations.invoke({
+		const invoke = () => this.endpoints.invocations.invoke({
 			endpointId: s.endpointId,
 			toolCallId: randomUUID(),
 			toolName: COMPUTER_CONTROL_TOOL,
@@ -466,6 +647,9 @@ var ComputerRuntime = class {
 			descriptorRevision: tool.revision,
 			signal
 		});
+		const phase = command.op === "open" ? "open" : command.op === "act" ? "dispatch" : command.op === "observe" ? "observe" : command.op === "release" ? "release" : void 0;
+		if (command.op === "observe") s.diagnostics?.count("observations");
+		const reply = s.diagnostics && phase ? await s.diagnostics.measure(phase, invoke) : await invoke();
 		const file = reply.content.find((item) => item.type === "file");
 		let frame;
 		if (file?.type === "file" && reply.invocationId) frame = this.endpoints.uploads.takeComputerFrame(file.fileId, reply.invocationId);
@@ -490,6 +674,8 @@ var ComputerRuntime = class {
 		if (this.sessions.get(s.owner) !== s) return;
 		this.sessions.delete(s.owner);
 		leaveComputerControl(s.owner);
+		s.diagnostics.finish(true);
+		this.retainReport(s.owner, s.diagnostics);
 		s.task.clear();
 		s.pendingContext = void 0;
 		s.controller.abort();
@@ -516,8 +702,39 @@ var ComputerRuntime = class {
 				return true;
 			}
 		}
+		s.diagnostics.fail(/* @__PURE__ */ new Error("COMPUTER_RELEASE_UNCONFIRMED"), "release", "unknown");
+		s.diagnostics.finish(false);
 		return false;
 	}
+	getReport(owner) {
+		this.pruneReports();
+		return (this.sessions.get(owner)?.diagnostics ?? this.openingReports.get(owner) ?? this.reports.get(owner)?.diagnostics)?.snapshot();
+	}
+	retainReport(owner, diagnostics) {
+		this.pruneReports();
+		const previous = this.reports.get(owner);
+		this.reports.delete(owner);
+		this.reports.set(owner, {
+			diagnostics,
+			expiresAt: Date.now() + 30 * 6e4
+		});
+		while (this.reports.size > 32) this.reports.delete(this.reports.keys().next().value);
+		if (previous?.diagnostics !== diagnostics) {
+			const report = diagnostics.snapshot();
+			log.info({
+				reportId: report.reportId,
+				conversationId: owner,
+				sessionId: report.sessionId,
+				status: report.status,
+				durationMs: report.durationMs,
+				counters: report.counters,
+				failure: report.failure
+			}, "Computer session diagnostics retained");
+		}
+	}
+	pruneReports() {
+		for (const [owner, report] of this.reports) if (report.expiresAt <= Date.now()) this.reports.delete(owner);
+	}
 	async shutdown() {
 		await Promise.all([...new Set([...this.sessions.keys(), ...this.operations.keys()])].map((owner) => this.close(owner)));
 	}

package/dist/src/computer/semantic-actions.d.ts

@@ -0,0 +1,3 @@
+import type { ComputerAction, ComputerObservation } from '@xopcai/computer-control-contract';
+/** Ground only an unambiguous single click; never reinterpret double clicks or text entry. */
+export declare function groundComputerAction(action: ComputerAction, observation: ComputerObservation): ComputerAction;

package/dist/src/computer/semantic-actions.js

@@ -0,0 +1,29 @@
+//#region src/computer/semantic-actions.ts
+/** Ground only an unambiguous single click; never reinterpret double clicks or text entry. */
+function groundComputerAction(action, observation) {
+	if (action.kind !== "click" || action.count !== 1 || action.point.x < 0 || action.point.y < 0 || action.point.x >= observation.imageWidth || action.point.y >= observation.imageHeight) return action;
+	const kind = action.button === "left" ? "activate" : "showMenu";
+	try {
+		const { elements, truncated } = JSON.parse(observation.summary);
+		if (!Array.isArray(elements) || truncated === true) return action;
+		const candidates = elements.filter((element) => {
+			if (!element || typeof element !== "object") return false;
+			const b = element.bounds;
+			return (Array.isArray(element.actions) && element.actions.some((name) => name === "AXPress" || name === "AXShowMenu") || element.semanticActions?.length || /^AX(Button|CheckBox|RadioButton|DisclosureTriangle|PopUpButton|ComboBox|TextField|TextArea|SecureTextField|Link|Row|Cell|Slider)$/.test(element.role)) && b && [
+				b.x,
+				b.y,
+				b.width,
+				b.height
+			].every(Number.isFinite) && b.width > 0 && b.height > 0 && action.point.x >= b.x && action.point.y >= b.y && action.point.x < b.x + b.width && action.point.y < b.y + b.height;
+		});
+		const element = candidates.length === 1 ? candidates[0] : void 0;
+		return element && typeof element.ref === "string" && element.enabled !== false && element.in_web_content !== true && Array.isArray(element.semanticActions) && element.semanticActions.includes(kind) ? {
+			kind,
+			ref: element.ref
+		} : action;
+	} catch {
+		return action;
+	}
+}
+//#endregion
+export { groundComputerAction };

package/dist/src/config/index.js

@@ -5,7 +5,7 @@ import { resolveAgentWorkspaceDir } from "../agent/agent-scope.js";
 import { extractProfileAgentId, resolveEffectiveAgentConfigForAgent, resolveEffectiveAgentConfigForSession, resolveEffectiveAgentProfile, resolveEffectiveAgentProfileForSession } from "./agent-profile.js";
 import { TelegramAccountConfigSchema, TelegramConfigSchema, TelegramGroupConfigSchema, TelegramTopicConfigSchema } from "../../extensions/telegram/src/config-schema.js";
 import { WeixinAccountConfigSchema, WeixinConfigSchema } from "../../extensions/weixin/src/config-schema.js";
-import { AgentImageGenerationModelSchema, AgentModelRefSchema, AgentModelsSchema, BrowserConfigSchema, ChannelsConfigSchema, CommandsConfigSchema, ConfigSchema, ConnectorsConfigSchema, ExperimentalConfigSchema, ExtensionSecurityConfigSchema, ExtensionSlotsConfigSchema, ExtensionsConfigSchema, GatewayAuthRateLimitSchema, GatewayAuthSchema, GatewayBindModeSchema, GatewayChannelConnectDeferModeSchema, GatewayConfigSchema, GatewayModeSchema, GatewayRemoteSchema, GatewayScenesConfigSchema, GatewaySecuritySchema, GatewayTailscaleConsentSchema, GatewayTailscaleSchema, GatewayTlsSchema, GatewayTrustedProxySchema, GatewayWebchatConfigSchema, McpConfigSchema, McpOAuthConfigSchema, McpServerSchema, MediaUnderstandingCapabilitiesSchema, MediaUnderstandingModelSchema, MessagesConfigSchema, ModelCatalogConfigSchema, ModelsDevConfigSchema, ProviderAuthConfigSchema, ProviderAzureConfigSchema, ProviderRequestOverridesSchema, ProvidersConfigSchema, RuntimeToolsConfigSchema, STTConfigSchema, STTFallbackConfigSchema, STTProviderConfigSchema, SearchProviderEntrySchema, SessionConfigSchema, SessionDmScopeSchema, SessionResetByTypeSchema, SessionResetConfigSchema, SessionResetModeSchema, SessionScopeSchema, SessionStorageConfigSchema, TTSConfigSchema, TTSEdgeConfigSchema, TTSFallbackConfigSchema, TTSModelOverridesConfigSchema, TTSProviderConfigSchema, TTSSummarizationConfigSchema, ToolsConfigSchema, ToolsMediaAudioConfigSchema, ToolsMediaConfigSchema, TunnelBrokerUrlSchema, TunnelConfigSchema, TunnelConsentSchema, UpdateAutoConfigSchema, UpdateConfigSchema, VoiceConfigSchema, VoiceRefinementConfigSchema, WebSearchConfigSchema, WebToolsConfigSchema, WebsiteBlocklistSchema, WorkspaceConfigSchema, WorkspaceImportConfigSchema, getAgentDefaultChatModelConfig, getAgentDefaultImageGenerationModelConfig, getAgentDefaultImageModelConfig, getAgentDefaultModelRef, init_schema, parseModelRef } from "./schema.js";
+import { AgentImageGenerationModelSchema, AgentModelRefSchema, AgentModelsSchema, BrowserConfigSchema, ChannelsConfigSchema, CommandsConfigSchema, ConfigSchema, ConnectorsConfigSchema, ExperimentalConfigSchema, ExtensionSecurityConfigSchema, ExtensionSlotsConfigSchema, ExtensionsConfigSchema, GatewayAuthRateLimitSchema, GatewayAuthSchema, GatewayBindModeSchema, GatewayChannelConnectDeferModeSchema, GatewayConfigSchema, GatewayModeSchema, GatewayRemoteSchema, GatewaySecuritySchema, GatewayTailscaleConsentSchema, GatewayTailscaleSchema, GatewayTlsSchema, GatewayTrustedProxySchema, GatewayWebchatConfigSchema, McpConfigSchema, McpOAuthConfigSchema, McpServerSchema, MediaUnderstandingCapabilitiesSchema, MediaUnderstandingModelSchema, MessagesConfigSchema, ModelCatalogConfigSchema, ModelsDevConfigSchema, ProviderAuthConfigSchema, ProviderAzureConfigSchema, ProviderRequestOverridesSchema, ProvidersConfigSchema, RuntimeToolsConfigSchema, STTConfigSchema, STTFallbackConfigSchema, STTProviderConfigSchema, SearchProviderEntrySchema, SessionConfigSchema, SessionDmScopeSchema, SessionResetByTypeSchema, SessionResetConfigSchema, SessionResetModeSchema, SessionScopeSchema, SessionStorageConfigSchema, TTSConfigSchema, TTSEdgeConfigSchema, TTSFallbackConfigSchema, TTSModelOverridesConfigSchema, TTSProviderConfigSchema, TTSSummarizationConfigSchema, ToolsConfigSchema, ToolsMediaAudioConfigSchema, ToolsMediaConfigSchema, TunnelBrokerUrlSchema, TunnelConfigSchema, TunnelConsentSchema, UpdateAutoConfigSchema, UpdateConfigSchema, VoiceConfigSchema, VoiceRefinementConfigSchema, WebSearchConfigSchema, WebToolsConfigSchema, WebsiteBlocklistSchema, WorkspaceConfigSchema, WorkspaceImportConfigSchema, getAgentDefaultChatModelConfig, getAgentDefaultImageGenerationModelConfig, getAgentDefaultImageModelConfig, getAgentDefaultModelRef, init_schema, parseModelRef } from "./schema.js";
 import { clearConfigValueCache, getAllowedCommands, getCacheStats, init_resolve_config_value, resolveConfigValue, resolveHeaders, testApiKeyResolution } from "./resolve-config-value.js";
 import { FILENAMES, WORKSPACE_FILES, existsSync, hasBundledExtensionManifest, init_paths, resolveAgentAuthProfilesPath, resolveAgentDir, resolveAgentHomeDir, resolveAgentProfileDir, resolveAgentProfileMarkdownPath, resolveAuthProfilesPath, resolveBinDir, resolveBundledExtensionsDir, resolveBundledSkillsDir, resolveConfigPath, resolveCredentialsDir, resolveExtensionSdkPath, resolveExtensionsDir, resolveExtensionsLockPath, resolveInboxDir, resolveInboxMessagePath, resolveInboxPendingDir, resolveInboxProcessedDir, resolveLogPath, resolveLogsDir, resolveModelsJsonPath, resolveNodeBinDir, resolveNodeBinPath, resolveNodeToolsDir, resolveNpmBinPath, resolveOAuthPath, resolvePidPath, resolveSkillPath, resolveSkillsCachePath, resolveSkillsDir, resolveSkillsLockPath, resolveSocketPath, resolveStatusPath, resolveToolsDir, resolveWorkspaceExtensionsDir, resolveWorkspaceFile, resolveWorkspaceStateDir, resolveWorkspaceStatePath, resolveXopcBinPath } from "./paths.js";
 import { CustomModelSchema, ImageGenerationCapabilitiesSchema, ImageGenerationModelSchema, ImageGenerationProviderSchema, ModelOverrideSchema, ModelsJsonSchema, OpenAICompatSchema, OpenAICompletionsCompatSchema, OpenAIResponsesCompatSchema, OpenRouterRoutingSchema, ProviderConfigSchema, VercelGatewayRoutingSchema, getDefaultModelValues, init_models_json, loadModelsJson, modelsJsonExists, saveModelsJson, validateModelsConfig } from "./models-json.js";
@@ -26,4 +26,4 @@ init_workspace_path();
 init_models_json();
 init_resolve_config_value();
 //#endregion
-export { AgentImageGenerationModelSchema, AgentModelRefSchema, AgentModelsSchema, BASE_RELOAD_RULES, BrowserConfigSchema, ChannelsConfigSchema, CommandsConfigSchema, ConfigHotReloader, ConfigSchema, ConnectorsConfigSchema, CustomModelSchema, ENV_VARS, ExperimentalConfigSchema, ExtensionSecurityConfigSchema, ExtensionSlotsConfigSchema, ExtensionsConfigSchema, FILENAMES, GatewayAuthRateLimitSchema, GatewayAuthSchema, GatewayBindModeSchema, GatewayChannelConnectDeferModeSchema, GatewayConfigSchema, GatewayModeSchema, GatewayRemoteSchema, GatewayScenesConfigSchema, GatewaySecuritySchema, GatewayTailscaleConsentSchema, GatewayTailscaleSchema, GatewayTlsSchema, GatewayTrustedProxySchema, GatewayWebchatConfigSchema, ImageGenerationCapabilitiesSchema, ImageGenerationModelSchema, ImageGenerationProviderSchema, McpConfigSchema, McpOAuthConfigSchema, McpServerSchema, MediaUnderstandingCapabilitiesSchema, MediaUnderstandingModelSchema, MessagesConfigSchema, ModelCatalogConfigSchema, ModelOverrideSchema, ModelsDevConfigSchema, ModelsJsonSchema, OpenAICompatSchema, OpenAICompletionsCompatSchema, OpenAIResponsesCompatSchema, OpenRouterRoutingSchema, ProfileManager, ProviderAuthConfigSchema, ProviderAzureConfigSchema, ProviderConfigSchema, ProviderRequestOverridesSchema, ProvidersConfigSchema, RuntimeToolsConfigSchema, STTConfigSchema, STTFallbackConfigSchema, STTProviderConfigSchema, SearchProviderEntrySchema, SessionConfigSchema, SessionDmScopeSchema, SessionResetByTypeSchema, SessionResetConfigSchema, SessionResetModeSchema, SessionScopeSchema, SessionStorageConfigSchema, TTSConfigSchema, TTSEdgeConfigSchema, TTSFallbackConfigSchema, TTSModelOverridesConfigSchema, TTSProviderConfigSchema, TTSSummarizationConfigSchema, TelegramAccountConfigSchema, TelegramConfigSchema, TelegramGroupConfigSchema, TelegramTopicConfigSchema, ToolsConfigSchema, ToolsMediaAudioConfigSchema, ToolsMediaConfigSchema, TunnelBrokerUrlSchema, TunnelConfigSchema, TunnelConsentSchema, UpdateAutoConfigSchema, UpdateConfigSchema, VercelGatewayRoutingSchema, VoiceConfigSchema, VoiceRefinementConfigSchema, WORKSPACE_FILES, WebSearchConfigSchema, WebToolsConfigSchema, WebsiteBlocklistSchema, WeixinAccountConfigSchema, WeixinConfigSchema, WorkspaceConfigSchema, WorkspaceImportConfigSchema, XOPC_DB_FILENAME, applyConfigOverrides, assertConfigRewriteSafeForRunningGateway, buildReloadPlan, canonicalizeConfiguredMcpServer, clearConfigValueCache, createProfile, deleteProfile, diffConfigPaths, existsSync, expandWorkspacePathString, extractProfileAgentId, formatThinkingLevels, getAgentDefaultChatModelConfig, getAgentDefaultImageGenerationModelConfig, getAgentDefaultImageModelConfig, getAgentDefaultModelRef, getAllowedCommands, getCacheStats, getConfigOverrides, getCurrentProfile, getDefaultModelValues, resolveModelsJsonPath as getModelsJsonPath, getProfileManager, getProfileNameFromDir, getSwitchCommand, getWorkspacePath, hasBundledExtensionManifest, listAgentProfileMarkdownDirs, listAgentWorkspaceDirs, listProfiles, listThinkingLevels, loadConfig, loadModelsJson, matchReloadRule, modelsJsonExists, normalizeConfiguredMcpServers, normalizeElevatedMode, normalizeReasoningLevel, normalizeThinkLevel, normalizeVerboseLevel, normalizeWorkspaceDir, parseModelRef, resetConfigOverrides, resolveAgentAuthProfilesPath, resolveAgentDir, resolveAgentHomeDir, resolveAgentProfileDir, resolveAgentProfileMarkdownPath, resolveAgentWorkspaceDir, resolveAuthProfilesPath, resolveBinDir, resolveBundledExtensionsDir, resolveBundledSkillsDir, resolveConfigPath, resolveConfigValue, resolveCredentialsDir, resolveDefaultAgentWorkspaceDir, resolveEffectiveAgentConfigForAgent, resolveEffectiveAgentConfigForSession, resolveEffectiveAgentProfile, resolveEffectiveAgentProfileForSession, resolveExtensionSdkPath, resolveExtensionsDir, resolveExtensionsLockPath, resolveHeaders, resolveHomeDir, resolveInboxDir, resolveInboxMessagePath, resolveInboxPendingDir, resolveInboxProcessedDir, resolveLogPath, resolveLogsDir, resolveModelsJsonPath, resolveNodeBinDir, resolveNodeBinPath, resolveNodeToolsDir, resolveNpmBinPath, resolveOAuthPath, resolvePidPath, resolveProfileStateDir, resolveSkillPath, resolveSkillsCachePath, resolveSkillsDir, resolveSkillsLockPath, resolveSocketPath, resolveStateDir, resolveStatusPath, resolveToolsDir, resolveWorkspaceExtensionsDir, resolveWorkspaceFile, resolveWorkspaceRoot, resolveWorkspaceStateDir, resolveWorkspaceStatePath, resolveXopcBinPath, resolveXopcCloudCatalogCachePath, resolveXopcDatabasePath, saveConfig, saveModelsJson, setConfigOverride, testApiKeyResolution, thinkLevelToNumber, unsetConfigOverride, validateModelsConfig };
+export { AgentImageGenerationModelSchema, AgentModelRefSchema, AgentModelsSchema, BASE_RELOAD_RULES, BrowserConfigSchema, ChannelsConfigSchema, CommandsConfigSchema, ConfigHotReloader, ConfigSchema, ConnectorsConfigSchema, CustomModelSchema, ENV_VARS, ExperimentalConfigSchema, ExtensionSecurityConfigSchema, ExtensionSlotsConfigSchema, ExtensionsConfigSchema, FILENAMES, GatewayAuthRateLimitSchema, GatewayAuthSchema, GatewayBindModeSchema, GatewayChannelConnectDeferModeSchema, GatewayConfigSchema, GatewayModeSchema, GatewayRemoteSchema, GatewaySecuritySchema, GatewayTailscaleConsentSchema, GatewayTailscaleSchema, GatewayTlsSchema, GatewayTrustedProxySchema, GatewayWebchatConfigSchema, ImageGenerationCapabilitiesSchema, ImageGenerationModelSchema, ImageGenerationProviderSchema, McpConfigSchema, McpOAuthConfigSchema, McpServerSchema, MediaUnderstandingCapabilitiesSchema, MediaUnderstandingModelSchema, MessagesConfigSchema, ModelCatalogConfigSchema, ModelOverrideSchema, ModelsDevConfigSchema, ModelsJsonSchema, OpenAICompatSchema, OpenAICompletionsCompatSchema, OpenAIResponsesCompatSchema, OpenRouterRoutingSchema, ProfileManager, ProviderAuthConfigSchema, ProviderAzureConfigSchema, ProviderConfigSchema, ProviderRequestOverridesSchema, ProvidersConfigSchema, RuntimeToolsConfigSchema, STTConfigSchema, STTFallbackConfigSchema, STTProviderConfigSchema, SearchProviderEntrySchema, SessionConfigSchema, SessionDmScopeSchema, SessionResetByTypeSchema, SessionResetConfigSchema, SessionResetModeSchema, SessionScopeSchema, SessionStorageConfigSchema, TTSConfigSchema, TTSEdgeConfigSchema, TTSFallbackConfigSchema, TTSModelOverridesConfigSchema, TTSProviderConfigSchema, TTSSummarizationConfigSchema, TelegramAccountConfigSchema, TelegramConfigSchema, TelegramGroupConfigSchema, TelegramTopicConfigSchema, ToolsConfigSchema, ToolsMediaAudioConfigSchema, ToolsMediaConfigSchema, TunnelBrokerUrlSchema, TunnelConfigSchema, TunnelConsentSchema, UpdateAutoConfigSchema, UpdateConfigSchema, VercelGatewayRoutingSchema, VoiceConfigSchema, VoiceRefinementConfigSchema, WORKSPACE_FILES, WebSearchConfigSchema, WebToolsConfigSchema, WebsiteBlocklistSchema, WeixinAccountConfigSchema, WeixinConfigSchema, WorkspaceConfigSchema, WorkspaceImportConfigSchema, XOPC_DB_FILENAME, applyConfigOverrides, assertConfigRewriteSafeForRunningGateway, buildReloadPlan, canonicalizeConfiguredMcpServer, clearConfigValueCache, createProfile, deleteProfile, diffConfigPaths, existsSync, expandWorkspacePathString, extractProfileAgentId, formatThinkingLevels, getAgentDefaultChatModelConfig, getAgentDefaultImageGenerationModelConfig, getAgentDefaultImageModelConfig, getAgentDefaultModelRef, getAllowedCommands, getCacheStats, getConfigOverrides, getCurrentProfile, getDefaultModelValues, resolveModelsJsonPath as getModelsJsonPath, getProfileManager, getProfileNameFromDir, getSwitchCommand, getWorkspacePath, hasBundledExtensionManifest, listAgentProfileMarkdownDirs, listAgentWorkspaceDirs, listProfiles, listThinkingLevels, loadConfig, loadModelsJson, matchReloadRule, modelsJsonExists, normalizeConfiguredMcpServers, normalizeElevatedMode, normalizeReasoningLevel, normalizeThinkLevel, normalizeVerboseLevel, normalizeWorkspaceDir, parseModelRef, resetConfigOverrides, resolveAgentAuthProfilesPath, resolveAgentDir, resolveAgentHomeDir, resolveAgentProfileDir, resolveAgentProfileMarkdownPath, resolveAgentWorkspaceDir, resolveAuthProfilesPath, resolveBinDir, resolveBundledExtensionsDir, resolveBundledSkillsDir, resolveConfigPath, resolveConfigValue, resolveCredentialsDir, resolveDefaultAgentWorkspaceDir, resolveEffectiveAgentConfigForAgent, resolveEffectiveAgentConfigForSession, resolveEffectiveAgentProfile, resolveEffectiveAgentProfileForSession, resolveExtensionSdkPath, resolveExtensionsDir, resolveExtensionsLockPath, resolveHeaders, resolveHomeDir, resolveInboxDir, resolveInboxMessagePath, resolveInboxPendingDir, resolveInboxProcessedDir, resolveLogPath, resolveLogsDir, resolveModelsJsonPath, resolveNodeBinDir, resolveNodeBinPath, resolveNodeToolsDir, resolveNpmBinPath, resolveOAuthPath, resolvePidPath, resolveProfileStateDir, resolveSkillPath, resolveSkillsCachePath, resolveSkillsDir, resolveSkillsLockPath, resolveSocketPath, resolveStateDir, resolveStatusPath, resolveToolsDir, resolveWorkspaceExtensionsDir, resolveWorkspaceFile, resolveWorkspaceRoot, resolveWorkspaceStateDir, resolveWorkspaceStatePath, resolveXopcBinPath, resolveXopcCloudCatalogCachePath, resolveXopcDatabasePath, saveConfig, saveModelsJson, setConfigOverride, testApiKeyResolution, thinkLevelToNumber, unsetConfigOverride, validateModelsConfig };