@xopcai/xopc 0.0.424 → 0.0.425

package/dist/src/computer/runtime.js

@@ -1,16 +1,18 @@
 import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
 import { resolveEffectiveAgentConfigForSession } from "../config/agent-profile.js";
-import { COMPUTER_CONTROL_TOOL, ComputerAppSchema, ComputerObservationSchema, ComputerWindowSchema, init_src } from "../../packages/computer-control-contract/src/index.js";
+import { COMPUTER_CONTROL_TOOL, ComputerActionSchema, ComputerAppSchema, ComputerObservationSchema, ComputerTransportTimingSchema, ComputerWindowSchema, init_src } from "../../packages/computer-control-contract/src/index.js";
 import { ComputerServiceLimitsSchema, computerModelProfile, init_model_policy } from "./model-policy.js";
 import { getApiKey, init_providers, resolveModel } from "../providers/index.js";
 import { rejectUnbudgetedCloudTransport } from "../providers/xopc-cloud-credit-budget.js";
 import { enterComputerControl, leaveComputerControl } from "./control-guard.js";
+import { clearComputerFrames, computerAgentTurn, retainComputerFrame } from "./agent-context.js";
 import { ComputerDiagnosticSchema, ComputerOperationError, computerDiagnostic, computerRecovery } from "./errors.js";
 import { createComputerModelAdapter, predictComputerStep, readComputerJson } from "./model-adapter.js";
 import { groundComputerAction } from "./semantic-actions.js";
 import { ComputerFlowCapture, ComputerFlowStore, computerFlowHint, computerFlowMetadata } from "./flows.js";
 import { ComputerNativeTimingSchema, ComputerSessionDiagnostics } from "./diagnostics.js";
+import { executeComputerProgram } from "./program-runtime.js";
 import { ComputerExpectationSchema, ComputerTaskState, verifyComputerExpectation } from "./task-state.js";
 import { randomUUID } from "node:crypto";
 import { z } from "zod";
@@ -36,6 +38,7 @@ const Reply = z.object({
 	grantId: z.string().optional(),
 	observation: ComputerObservationSchema.optional(),
 	nativeMs: ComputerNativeTimingSchema.optional(),
+	transportTiming: ComputerTransportTimingSchema.optional(),
 	errorCode: z.string().optional(),
 	diagnostic: ComputerDiagnosticSchema.optional(),
 	apps: z.array(ComputerAppSchema).max(100).optional(),
@@ -72,7 +75,8 @@ const ComputerUseInputSchema = z.discriminatedUnion("op", [
 		appRef: z.string().min(1).max(200),
 		windowRef: z.string().min(1).max(200).optional(),
 		mode: z.enum(["observe", "control"]),
-		prepare: z.boolean()
+		prepare: z.boolean(),
+		planner: z.enum(["gui", "agent"]).optional()
 	}).strict(),
 	z.object({
 		op: z.literal("observe"),
@@ -85,6 +89,12 @@ const ComputerUseInputSchema = z.discriminatedUnion("op", [
 		expect: ComputerExpectationSchema.optional(),
 		flowId: z.uuid().optional()
 	}).strict(),
+	z.object({
+		op: z.literal("exec"),
+		code: z.string().max(65536),
+		maxActions: z.number().int().min(1).max(12).optional(),
+		timeoutMs: z.number().int().min(100).max(12e4).optional()
+	}).strict(),
 	z.object({
 		op: z.literal("run"),
 		goal: z.string().min(1).max(4e3),
@@ -113,7 +123,7 @@ var ComputerRuntime = class {
 		this.getTurnEndpoint = getTurnEndpoint;
 		this.flowStore = flowStore;
 	}
-	async execute(owner, input, signal) {
+	async execute(owner, input, signal, execution) {
 		input = ComputerUseInputSchema.parse(input);
 		if (!owner) throw new Error("COMPUTER_CONVERSATION_REQUIRED");
 		if (input.op === "flows" || input.op === "forget_flow") {
@@ -207,7 +217,7 @@ var ComputerRuntime = class {
 		if (diagnostics) this.openingReports.set(owner, diagnostics);
 		try {
 			const operationSignal = signal ? AbortSignal.any([signal, operation.signal]) : operation.signal;
-			const result = input.op === "run" ? await this.executeGoal(owner, input, operationSignal, diagnostics) : await this.executeInput(owner, input, operationSignal, diagnostics);
+			const result = input.op === "exec" ? await this.executeProgram(owner, input, operationSignal, execution) : input.op === "run" ? await this.executeGoal(owner, input, operationSignal, diagnostics) : await this.executeInput(owner, input, operationSignal, diagnostics);
 			if (result.errorCode) diagnostics?.fail(new Error(result.errorCode), diagnostics.phase, result.receipt?.dispatch ?? "notStarted");
 			if (diagnostics && !this.sessions.has(owner)) this.retainReport(owner, diagnostics);
 			return {
@@ -242,6 +252,182 @@ var ComputerRuntime = class {
 		const effective = resolveEffectiveAgentConfigForSession(owner).config;
 		return JSON.stringify([effective.id, effective.workspace]);
 	}
+	async executeProgram(owner, input, signal, execution) {
+		const s = this.sessions.get(owner);
+		if (!s) throw new Error("COMPUTER_OPEN_REQUIRED");
+		if (!("planner" in s.binding) || s.binding.planner !== "agent" || !execution) throw new Error("COMPUTER_PLANNER_MISMATCH");
+		const turn = computerAgentTurn(owner);
+		if (JSON.stringify(turn.binding) !== JSON.stringify(s.binding)) {
+			await this.close(owner);
+			throw new Error("COMPUTER_AGENT_MODEL_CHANGED");
+		}
+		const combined = AbortSignal.any([signal, s.controller.signal]);
+		let current;
+		let actions = 0;
+		let imageRef;
+		const observe = async (observationSignal = combined) => {
+			current?.frame?.fill(0);
+			current = await this.send(s, {
+				op: "observe",
+				sessionId: s.id,
+				owner
+			}, observationSignal);
+			if (current.value.status !== "ready" || current.value.errorCode || !current.value.observation) throw new Error(current.value.errorCode ?? "COMPUTER_NOT_READY");
+			const observation = current.value.observation;
+			let data;
+			try {
+				data = JSON.parse(observation.summary);
+			} catch {
+				data = {
+					text: observation.summary,
+					elements: [],
+					truncated: true
+				};
+			}
+			return {
+				snapshotId: observation.id,
+				...data
+			};
+		};
+		const snapshot = (id) => {
+			const observation = current?.value.observation;
+			if (!observation || observation.id !== id || Date.now() - observation.capturedAt > 5e3) throw new Error("COMPUTER_STALE_OBSERVATION");
+			return observation;
+		};
+		try {
+			const result = await executeComputerProgram(input.code, async (method, args, callSignal) => {
+				callSignal.throwIfAborted();
+				if (!this.getConfig().computer.enabled || Date.now() - s.createdAt >= this.getConfig().computer.maxSessionDurationMs) throw new Error("COMPUTER_SESSION_BUDGET");
+				if (JSON.stringify(computerAgentTurn(owner).binding) !== JSON.stringify(s.binding)) throw new Error("COMPUTER_AGENT_MODEL_CHANGED");
+				await turn.authorize?.(method, args);
+				const hostResult = await (async () => {
+					if (method === "observe") {
+						const options = z.object({ visual: z.boolean().optional() }).strict().parse(args ?? {});
+						const state = await observe(callSignal);
+						if (options.visual && current?.frame && current.mimeType) imageRef = retainComputerFrame(owner, execution.toolCallId, current.frame, current.mimeType);
+						return state;
+					}
+					if (method === "emitScreenshot") {
+						snapshot(z.object({ snapshotId: z.string() }).strict().parse(args).snapshotId);
+						if (!current?.frame || !current.mimeType) throw new Error("COMPUTER_FRAME_REQUIRED");
+						imageRef = retainComputerFrame(owner, execution.toolCallId, current.frame, current.mimeType);
+						return { imageRef };
+					}
+					if (method === "resolve") {
+						const value = z.object({
+							snapshotId: z.string(),
+							role: z.string().max(100),
+							label: z.string().max(300)
+						}).strict().parse(args);
+						const data = JSON.parse(snapshot(value.snapshotId).summary);
+						if (data.truncated || !Array.isArray(data.elements)) throw new Error("COMPUTER_ELEMENT_REF_REQUIRED");
+						const candidates = data.elements.filter((e) => e.role === value.role && e.label === value.label && e.in_web_content !== true);
+						if (candidates.length !== 1 || typeof candidates[0].ref !== "string") throw new Error("COMPUTER_ELEMENT_REF_REQUIRED");
+						return candidates[0];
+					}
+					if (method === "verify" || method === "waitFor") {
+						const value = z.object({
+							expect: ComputerExpectationSchema,
+							timeoutMs: z.number().int().min(0).max(5e3).optional()
+						}).strict().parse(args);
+						const deadline = Date.now() + (method === "waitFor" ? value.timeoutMs ?? 2e3 : 0);
+						for (let sample = 0; sample < 20; sample++) {
+							const state = await observe(callSignal);
+							const verification = verifyComputerExpectation(value.expect, current.value.observation);
+							s.diagnostics.verify(verification);
+							if (verification?.status === "satisfied" || Date.now() >= deadline || sample === 19) return {
+								...state,
+								verification
+							};
+							await setTimeout(100, void 0, { signal: callSignal });
+						}
+					}
+					if (method === "act") {
+						if (s.mode === "observe") throw new Error("COMPUTER_READ_ONLY");
+						const value = z.object({
+							snapshotId: z.string(),
+							action: ComputerActionSchema,
+							expect: ComputerExpectationSchema.optional()
+						}).strict().parse(args);
+						const before = snapshot(value.snapshotId);
+						if (++actions > (input.maxActions ?? 6)) throw new Error("COMPUTER_ACTION_BUDGET");
+						const action = groundComputerAction(value.action, before);
+						s.task.assertProgress(action, before);
+						s.pendingContext = {
+							goal: "Authorized main-agent UI program",
+							before,
+							expectation: value.expect
+						};
+						const envelope = {
+							actionId: randomUUID(),
+							sessionId: s.id,
+							observationId: before.id,
+							brokerEpoch: before.brokerEpoch,
+							generation: before.generation,
+							grantId: current.value.grantId,
+							deadlineAt: Date.now() + 12e4,
+							action
+						};
+						const context = {
+							observationsReused: 0,
+							stabilityObservations: 0,
+							actionsRemaining: 1,
+							nativeMs: {
+								precheckMs: 0,
+								inputMs: 0,
+								postObserveMs: 0
+							}
+						};
+						clearComputerFrames(owner);
+						imageRef = void 0;
+						current?.frame?.fill(0);
+						current = void 0;
+						const authorizationDeadline = Date.now() + 6e4;
+						let reply = await this.dispatchAction(s, envelope, callSignal, context);
+						while (reply.status === "pending_action") {
+							if (Date.now() >= authorizationDeadline) throw new Error("COMPUTER_AUTHORIZATION_TIMEOUT");
+							await setTimeout(250, void 0, { signal: callSignal });
+							reply = await this.dispatchAction(s, envelope, callSignal, context);
+						}
+						current = context.observation;
+						if (reply.errorCode || reply.receipt?.dispatch !== "completed") throw new Error(reply.errorCode ?? "COMPUTER_NATIVE_ACTION_FAILED");
+						let state = {};
+						try {
+							state = JSON.parse(current?.value.observation?.summary ?? "{}");
+						} catch {
+							state = { truncated: true };
+						}
+						return {
+							...state,
+							receipt: reply.receipt,
+							verification: reply.verification,
+							snapshotId: current?.value.observation?.id
+						};
+					}
+					throw new Error("COMPUTER_INVALID_INPUT");
+				})();
+				return await turn.complete?.(method, hostResult) ?? hostResult;
+			}, s.programStore ??= Object.create(null), combined, input.timeoutMs);
+			if (result.status === "program_error") {
+				await this.close(owner);
+				return {
+					...result,
+					sessionId: s.id
+				};
+			}
+			return {
+				...result,
+				sessionId: s.id,
+				...imageRef ? { imageRef } : {},
+				budget: this.budget(s)
+			};
+		} catch (error) {
+			await this.close(owner);
+			throw error;
+		} finally {
+			current?.frame?.fill(0);
+		}
+	}
 	async executeGoal(owner, input, signal, diagnostics) {
 		const startedAt = Date.now();
 		const phasesBefore = diagnostics?.snapshot().phaseMs;
@@ -310,7 +496,7 @@ var ComputerRuntime = class {
 		} finally {
 			context.observation?.frame?.fill(0);
 			context.observation = void 0;
-			this.sessions.get(owner)?.adapter.reset?.();
+			this.sessions.get(owner)?.adapter?.reset?.();
 		}
 	}
 	async executeInput(owner, input, signal, diagnostics, goalContext) {
@@ -321,7 +507,7 @@ var ComputerRuntime = class {
 			throw new Error("COMPUTER_DISABLED");
 		}
 		let s = this.sessions.get(owner);
-		if (s?.binding.modelRef.startsWith("xopc-cloud/")) rejectUnbudgetedCloudTransport("computer use");
+		if (s && !("planner" in s.binding) && s.binding.modelRef.startsWith("xopc-cloud/")) rejectUnbudgetedCloudTransport("computer use");
 		if (input.op === "discover") {
 			const endpoint = this.resolveEndpoint(owner);
 			const id = randomUUID();
@@ -340,49 +526,66 @@ var ComputerRuntime = class {
 		if (input.op === "open" && !s) {
 			const endpoint = this.resolveEndpoint(owner);
 			const effective = resolveEffectiveAgentConfigForSession(owner).config;
-			const ref = effective.models.computerUse?.primary;
-			if (!ref) throw new Error("COMPUTER_MODEL_REQUIRED");
-			const model = resolveModel(ref);
-			const profile = computerModelProfile(model);
-			if (!profile) throw new Error("COMPUTER_MODEL_CAPABILITY_REQUIRED");
-			const apiKey = await getApiKey(model.provider, {
-				agentId: effective.id,
-				appConfig: config
-			});
-			if (!apiKey) throw new Error("COMPUTER_MODEL_KEY_MISSING");
-			let deployment;
+			let adapter;
+			let binding;
 			let serviceLimits;
-			if (model.provider === "xopc-cloud") {
-				rejectUnbudgetedCloudTransport("computer use");
-				const response = await fetch(`${model.baseUrl.replace(/\/$/, "")}/models`, {
-					headers: { Authorization: `Bearer ${apiKey}` },
-					redirect: "error",
-					signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(15e3)]) : AbortSignal.timeout(15e3)
+			if (input.planner === "agent") {
+				const turn = computerAgentTurn(owner);
+				binding = turn.binding;
+				turn.cleanup = async () => {
+					await this.close(owner);
+				};
+			} else {
+				const ref = effective.models.computerUse?.primary;
+				if (!ref) throw new Error("COMPUTER_MODEL_REQUIRED");
+				const model = resolveModel(ref);
+				const profile = computerModelProfile(model);
+				if (!profile) throw new Error("COMPUTER_MODEL_CAPABILITY_REQUIRED");
+				const apiKey = await getApiKey(model.provider, {
+					agentId: effective.id,
+					appConfig: config
 				});
-				if (!response.ok) {
-					await response.body?.cancel();
-					throw new Error("COMPUTER_DEPLOYMENT_UNAVAILABLE");
+				if (!apiKey) throw new Error("COMPUTER_MODEL_KEY_MISSING");
+				let deployment;
+				if (model.provider === "xopc-cloud") {
+					rejectUnbudgetedCloudTransport("computer use");
+					const response = await fetch(`${model.baseUrl.replace(/\/$/, "")}/models`, {
+						headers: { Authorization: `Bearer ${apiKey}` },
+						redirect: "error",
+						signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(15e3)]) : AbortSignal.timeout(15e3)
+					});
+					if (!response.ok) {
+						await response.body?.cancel();
+						throw new Error("COMPUTER_DEPLOYMENT_UNAVAILABLE");
+					}
+					const metadata = (await readComputerJson(response, 2 * 1024 * 1024)).data?.find((item) => item.id === model.id)?.xopc;
+					const parsedDeployment = z.object({
+						revision: z.string().regex(/^[a-f0-9]{64}$/),
+						origin: z.string().url().optional()
+					}).safeParse(metadata?.computerDeployment);
+					const parsedLimits = ComputerServiceLimitsSchema.optional().safeParse(metadata?.computerLimits);
+					if (!parsedDeployment.success || !parsedLimits.success) throw new Error("COMPUTER_DEPLOYMENT_UNAVAILABLE");
+					deployment = parsedDeployment.data;
+					serviceLimits = parsedLimits.data;
 				}
-				const metadata = (await readComputerJson(response, 2 * 1024 * 1024)).data?.find((item) => item.id === model.id)?.xopc;
-				const parsedDeployment = z.object({
-					revision: z.string().regex(/^[a-f0-9]{64}$/),
-					origin: z.string().url().optional()
-				}).safeParse(metadata?.computerDeployment);
-				const parsedLimits = ComputerServiceLimitsSchema.optional().safeParse(metadata?.computerLimits);
-				if (!parsedDeployment.success || !parsedLimits.success) throw new Error("COMPUTER_DEPLOYMENT_UNAVAILABLE");
-				deployment = parsedDeployment.data;
-				serviceLimits = parsedLimits.data;
+				adapter = createComputerModelAdapter({
+					modelId: model.id,
+					baseUrl: model.baseUrl,
+					api: model.api,
+					apiKey,
+					profile,
+					headers: model.headers,
+					deploymentRevision: deployment?.revision,
+					maxOutputTokens: Math.min(2048, model.maxTokens ?? 2048, serviceLimits?.maxOutputTokens ?? 2048)
+				});
+				binding = {
+					modelRef: ref,
+					profile,
+					origin: new URL(model.baseUrl).origin,
+					runtimeLocation: "unknown",
+					...deployment?.origin ? { upstreamOrigin: deployment.origin } : {}
+				};
 			}
-			const adapter = createComputerModelAdapter({
-				modelId: model.id,
-				baseUrl: model.baseUrl,
-				api: model.api,
-				apiKey,
-				profile,
-				headers: model.headers,
-				deploymentRevision: deployment?.revision,
-				maxOutputTokens: Math.min(2048, model.maxTokens ?? 2048, serviceLimits?.maxOutputTokens ?? 2048)
-			});
 			signal.throwIfAborted();
 			s = {
 				id: randomUUID(),
@@ -393,13 +596,7 @@ var ComputerRuntime = class {
 				windowRef: input.windowRef,
 				endpointId: endpoint.endpointId,
 				adapter,
-				binding: {
-					modelRef: ref,
-					profile,
-					origin: new URL(model.baseUrl).origin,
-					runtimeLocation: "unknown",
-					...deployment?.origin ? { upstreamOrigin: deployment.origin } : {}
-				},
+				binding,
 				createdAt: Date.now(),
 				actions: 0,
 				modelRequests: 0,
@@ -430,7 +627,7 @@ var ComputerRuntime = class {
 		s.diagnostics.phase = "observe";
 		try {
 			if (input.op === "open") {
-				if (input.appRef !== s.appRef || input.windowRef !== s.windowRef || input.mode !== s.mode || input.prepare !== s.prepare) throw new Error("COMPUTER_CLOSE_BEFORE_TARGET_CHANGE");
+				if (input.planner === "agent" !== "planner" in s.binding || input.appRef !== s.appRef || input.windowRef !== s.windowRef || input.mode !== s.mode || input.prepare !== s.prepare) throw new Error("COMPUTER_CLOSE_BEFORE_TARGET_CHANGE");
 				const reply = await this.send(s, {
 					op: "open",
 					sessionId: s.id,
@@ -474,7 +671,7 @@ var ComputerRuntime = class {
 				this.reserveRescue(s, signal);
 			} else if (status.value.status !== "ready") return this.present(status.value);
 			if (input.op === "step") {
-				if (input.flowId || s.lastFlowId) s.adapter.reset?.();
+				if (input.flowId || s.lastFlowId) s.adapter?.reset?.();
 				s.lastFlowId = input.flowId;
 			}
 			while (true) {
@@ -513,7 +710,7 @@ var ComputerRuntime = class {
 					};
 					if (input.op === "step" && input.expect?.kind !== "text" && verification?.status === "satisfied") {
 						s.diagnostics.verify(verification);
-						s.adapter.reset?.();
+						s.adapter?.reset?.();
 						return {
 							status: "condition_satisfied",
 							sessionId: s.id,
@@ -536,6 +733,7 @@ var ComputerRuntime = class {
 					}
 					phase = "model";
 					const predictionInput = input;
+					if (!s.adapter) throw new Error("COMPUTER_PLANNER_MISMATCH");
 					const proposal = await s.diagnostics.measure("model", () => predictComputerStep(s.adapter, {
 						goal: predictionInput.op === "observe" ? predictionInput.question : predictionInput.goal,
 						readOnly: predictionInput.op === "observe",
@@ -636,7 +834,7 @@ var ComputerRuntime = class {
 		}
 		s.rescues++;
 		s.diagnostics.count("rescueAttempts");
-		s.adapter.reset?.();
+		s.adapter?.reset?.();
 	}
 	async dispatchAction(s, envelope, signal, goalContext) {
 		if (!s.pending) {
@@ -655,7 +853,7 @@ var ComputerRuntime = class {
 			reply.frame?.fill(0);
 			throw new Error("COMPUTER_RECEIPT_MISMATCH");
 		}
-		if (goalContext && s.adapter.hasPendingPlan && reply.value.status === "ready" && !reply.value.errorCode && reply.value.receipt?.dispatch === "completed" && reply.value.observation) {
+		if (goalContext && s.adapter?.hasPendingPlan && reply.value.status === "ready" && !reply.value.errorCode && reply.value.receipt?.dispatch === "completed" && reply.value.observation) {
 			const receipt = reply.value.receipt;
 			const nativeMs = reply.value.nativeMs;
 			let stable = false;
@@ -686,7 +884,7 @@ var ComputerRuntime = class {
 					if (stable || next.value.status !== "ready" || next.value.errorCode) break;
 				}
 				if (!stable && !reply.value.errorCode) reply.value.errorCode = "COMPUTER_UI_UNSTABLE";
-			} catch (error) {
+			} catch {
 				reply.frame?.fill(0);
 				reply = { value: {
 					...reply.value,
@@ -800,7 +998,9 @@ var ComputerRuntime = class {
 		});
 		const phase = command.op === "open" ? "open" : command.op === "act" ? "dispatch" : command.op === "observe" ? "observe" : command.op === "release" ? "release" : void 0;
 		if (command.op === "observe") s.diagnostics?.count("observations");
+		const startedAt = performance.now();
 		const reply = s.diagnostics && phase ? await s.diagnostics.measure(phase, invoke) : await invoke();
+		const endpointRoundTripMs = performance.now() - startedAt;
 		const file = reply.content.find((item) => item.type === "file");
 		let frame;
 		if (file?.type === "file" && reply.invocationId) frame = this.endpoints.uploads.takeComputerFrame(file.fileId, reply.invocationId);
@@ -809,6 +1009,13 @@ var ComputerRuntime = class {
 			const json = reply.content.find((item) => item.type === "json");
 			const value = Reply.parse(json?.type === "json" ? json.value : void 0);
 			if (value.sessionId !== s.id) throw new Error("COMPUTER_SESSION_MISMATCH");
+			if (command.op === "act" && value.receipt?.actionId === command.envelope.actionId && (value.nativeMs || value.transportTiming)) s.diagnostics?.timing({
+				actionId: value.receipt.actionId,
+				kind: command.envelope.action.kind,
+				nativeMs: value.nativeMs,
+				transport: value.transportTiming,
+				endpointRoundTripMs
+			});
 			if (file && !frame || value.observation && !frame && !value.errorCode) throw new Error("COMPUTER_FRAME_HANDOFF_FAILED");
 			if (frame && (!value.observation || !["observe", "act"].includes(command.op))) throw new Error("COMPUTER_UNEXPECTED_FRAME");
 			return {
@@ -827,14 +1034,18 @@ var ComputerRuntime = class {
 		leaveComputerControl(s.owner);
 		s.diagnostics.finish(true);
 		this.retainReport(s.owner, s.diagnostics);
+		clearComputerFrames(s.owner);
+		s.programStore = void 0;
 		s.task.clear();
 		s.pendingContext = void 0;
 		s.controller.abort();
 	}
 	async close(owner) {
 		this.operations.get(owner)?.abort();
+		clearComputerFrames(owner);
 		const s = this.sessions.get(owner);
 		if (!s) return true;
+		s.programStore = void 0;
 		s.task.clear();
 		s.pendingContext = void 0;
 		s.controller.abort();

package/dist/src/config/schema.d.ts

@@ -1785,6 +1785,24 @@ export declare const ConfigSchema: z.ZodDefault<z.ZodObject<{
             direct: "direct";
         }>>;
         adbPath: z.ZodOptional<z.ZodString>;
+        autoGlmPrompt: z.ZodDefault<z.ZodEnum<{
+            current: "current";
+            upstream: "upstream";
+        }>>;
+        autoGlmProtocol: z.ZodDefault<z.ZodEnum<{
+            current: "current";
+            upstream: "upstream";
+        }>>;
+        autoGlmSampling: z.ZodDefault<z.ZodEnum<{
+            current: "current";
+            upstream: "upstream";
+        }>>;
+        modelResponseMode: z.ZodDefault<z.ZodEnum<{
+            json: "json";
+            stream: "stream";
+        }>>;
+        batchReadings: z.ZodDefault<z.ZodBoolean>;
+        modelRepetitionGuard: z.ZodDefault<z.ZodBoolean>;
         semanticControls: z.ZodDefault<z.ZodBoolean>;
         hierarchicalPlanning: z.ZodDefault<z.ZodBoolean>;
         maxActionsPerSession: z.ZodDefault<z.ZodNumber>;

package/dist/src/gateway/global-defaults-admin.js

@@ -3,8 +3,8 @@ import { init_agent_config } from "../agent-config/index.js";
 import { AgentCatalogRepository, init_repository } from "../agent-catalog/repository.js";
 import { validatePhoneModelChange } from "../phone/model-adapter.js";
 import { AgentCatalogService } from "../agent-catalog/service.js";
-import { validateComputerModelChange } from "../computer/model-config.js";
 import { GATEWAY_BUILTIN_TOOLS } from "./agent-builtin-tools.js";
+import { validateComputerModelChange } from "../computer/model-config.js";
 import { isDeepStrictEqual } from "node:util";
 //#region src/gateway/global-defaults-admin.ts
 init_agent_config();

package/dist/src/gateway/service/config-coordinator.d.ts

@@ -326,6 +326,12 @@ export declare class GatewayConfigCoordinator {
         phone: {
             enabled: boolean;
             executionMode: "collaborative" | "direct";
+            autoGlmPrompt: "current" | "upstream";
+            autoGlmProtocol: "current" | "upstream";
+            autoGlmSampling: "current" | "upstream";
+            modelResponseMode: "json" | "stream";
+            batchReadings: boolean;
+            modelRepetitionGuard: boolean;
             semanticControls: boolean;
             hierarchicalPlanning: boolean;
             maxActionsPerSession: number;

package/dist/src/phone/adb-driver.d.ts

@@ -1,3 +1,5 @@
+import { PhoneUiSession } from './ui-session.js';
+import { type PhoneTextProvider } from './text-session.js';
 import { type PhoneApplication, type PhoneTextInputStatus, type DiscoveredPhone, type PhoneAction, type PhoneElement, type PhoneObservation } from './contracts.js';
 import { runAdb } from './adb-command.js';
 export { runAdb } from './adb-command.js';
@@ -10,6 +12,7 @@ export interface PhoneDriver {
     applications?(serial: string, signal: AbortSignal): Promise<PhoneApplication[]>;
     textInputStatus?(serial: string, signal: AbortSignal): Promise<PhoneTextInputStatus>;
     observe(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
+    releaseInput?(): Promise<void>;
     capture?(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     observeForText?(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     execute(serial: string, action: PhoneAction, observation: PhoneObservation, signal: AbortSignal, options?: {
@@ -27,16 +30,27 @@ export declare class AdbPhoneDriver implements PhoneDriver {
     private readonly readApplications?;
     private readonly pasteText?;
     private readonly readHierarchy?;
-    constructor(adbPath?: string, command?: typeof runAdb, useAndroidCli?: boolean, readApplications?: (signal: AbortSignal) => Promise<PhoneApplication[]>, pasteText?: (text: string, signal: AbortSignal) => Promise<void>, readHierarchy?: (signal: AbortSignal) => Promise<string>);
+    private readonly createTextProvider?;
+    private readonly createObserver?;
+    private observers;
+    private openingObservers;
+    private observerEpoch;
+    private observer;
+    private textSessions;
+    releaseInput(): Promise<void>;
+    private paste;
+    constructor(adbPath?: string, command?: typeof runAdb, useAndroidCli?: boolean, readApplications?: (signal: AbortSignal) => Promise<PhoneApplication[]>, pasteText?: (text: string, signal: AbortSignal) => Promise<void>, readHierarchy?: (signal: AbortSignal) => Promise<string>, createTextProvider?: (serial: string, failed: () => void) => Promise<PhoneTextProvider>, createObserver?: (signal: AbortSignal) => Promise<PhoneUiSession>);
     textInputStatus(serial: string, signal: AbortSignal): Promise<PhoneTextInputStatus>;
     discover(signal?: AbortSignal): Promise<DiscoveredPhone[]>;
     applications(serial: string, signal: AbortSignal): Promise<PhoneApplication[]>;
     foreground(serial: string, signal: AbortSignal): Promise<string>;
     private shell;
     observe(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
+    private hierarchy;
     observeForText(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     /** Pixel capture avoids accessibility dumps on the human interaction path. */
     capture(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
+    private captureRaw;
     private confirmText;
     execute(serial: string, action: PhoneAction, observation: PhoneObservation, signal: AbortSignal, options?: {
         settleMs: number;