@onkernel/cua-agent 0.3.2 → 0.3.3

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.3.3 - 2026-06-12
+
+- The action translator now consumes the canonical `CuaAction` union with an
+  exhaustive switch. Malformed action shapes fail loudly instead of silently
+  coercing (previously e.g. a click at 0,0); the documented mouse-button
+  coercion to `"left"` is unchanged.
+- `prepareNextTurn` no longer rebuilds the turn context on every turn: it
+  keeps stock pi behavior until a user hook returns an update or a mid-run
+  model assignment requires a refresh.
+- One translator instance per runtime is shared between the executor tools
+  and the provider screenshot capability.
+- The `CuaAgentHarness` README quickstart showcases session-backed turns and
+  mid-session model switching; `computerUseExtra` is documented with its
+  rationale.
+- Update the `@onkernel/cua-ai` dependency to 0.3.0.
+
 ## 0.3.2 - 2026-06-11
 
 - Update the `@onkernel/cua-ai` dependency to 0.2.2.

package/README.md

@@ -37,11 +37,17 @@ await agent.prompt("Open news.ycombinator.com and summarize the top story.");
 
 ## Quick Start (`CuaAgentHarness`)
 
+`prompt()` returns the turn's final assistant message, and every turn is
+persisted to the session — later prompts see the full transcript. Runtime
+config like the model can change between turns (or even mid-turn, applying at
+the next provider request):
+
 ```ts
 import { CuaAgentHarness, InMemorySessionRepo, NodeExecutionEnv } from "@onkernel/cua-agent";
+import type { AssistantMessage } from "@onkernel/cua-ai";
 
 const sessionRepo = new InMemorySessionRepo();
-const session = await sessionRepo.create({ id: "example" });
+const session = await sessionRepo.create({ id: "research" });
 
 const harness = new CuaAgentHarness({
   browser,
@@ -51,22 +57,26 @@ const harness = new CuaAgentHarness({
   session,
 });
 
-const response = await harness.prompt("Open example.com and tell me the current URL.");
-const branch = await session.getBranch();
-const lastAssistant = [...branch]
-  .reverse()
-  .flatMap((entry) =>
-    entry.type === "message" && entry.message.role === "assistant" ? [entry.message] : [],
-  )[0];
-const assistant = lastAssistant ?? response;
-const assistantText = assistant.content
-  .flatMap((block) => (block.type === "text" ? [block.text] : []))
-  .join("")
-  .trim();
-console.log("assistant stopReason:", assistant.stopReason);
-console.log("assistant text:", assistantText || "(no text)");
+const textOf = (message: AssistantMessage) =>
+  message.content.flatMap((block) => (block.type === "text" ? [block.text] : [])).join("").trim();
+
+// Turn 1: a session-backed prompt.
+const first = await harness.prompt("Open example.com and describe what you see.");
+console.log(textOf(first));
+
+// Swap providers mid-session; CUA tools and the default prompt refresh to match.
+await harness.setModel("anthropic:claude-opus-4-7");
+
+// Turn 2 continues the same transcript on the new model.
+const second = await harness.prompt("Open the most relevant link from what you found.");
+console.log(textOf(second));
 ```
 
+While a turn is running, `steer()` injects course corrections, `followUp()`
+queues the next instruction, and `subscribe()` streams the underlying agent
+events. `compact()` and session branching are available for long-running
+transcripts — see the pi-agent-core docs for the full harness lifecycle.
+
 Use `CuaAgent` when you want direct pi `Agent` control: raw message state,
 lifecycle events, custom streaming, and explicit prompt/continue/queue control.
 Reach for the harness shape when you want an app layer around the loop:
@@ -108,9 +118,11 @@ computer-use tools. This is useful when the model needs to call
 application-specific code, such as looking up a record, writing a database row,
 or handing off to another service while it also controls the browser.
 
-`computerUseExtra: true` adds the `computer_use_extra` tool. Use it when you
-want one compact helper for common browser navigation/read operations:
-`goto`, `back`, `forward`, and `url`.
+Not every provider's native computer-use vocabulary includes browser
+navigation — some models can click and type but have no direct way to open a
+URL or go back. `computerUseExtra: true` adds `computer_use_extra`, a
+provider-neutral escape hatch exposing `goto`, `back`, `forward`, and `url`
+so navigation works uniformly regardless of which model is driving.
 
 ### Model Switching
 

package/dist/index.d.ts

@@ -102,7 +102,9 @@ type CuaAgentHarnessOptions<TSkill extends Skill = Skill, TPromptTemplate extend
 declare class CuaAgent extends Agent {
   private readonly runtime;
   private readonly ownsSystemPrompt;
+  private runtimeDirty;
   private stateProxy?;
+  private stateProxyTarget?;
   constructor(options: CuaAgentOptions);
   /**
    * Return a state proxy so `agent.state.model = "provider:model"` can behave

package/dist/index.js

@@ -156,8 +156,8 @@ var InternalComputerTranslator = class {
 	async currentMousePosition() {
 		const pos = await this.client.browsers.computer.getMousePosition(this.sessionId);
 		return {
-			x: toInt(pos.x),
-			y: toInt(pos.y)
+			x: Math.trunc(pos.x),
+			y: Math.trunc(pos.y)
 		};
 	}
 	async executeBatch(actions) {
@@ -167,236 +167,196 @@ var InternalComputerTranslator = class {
 			if (pending.length === 0) return;
 			await this.runKernelBatch(pending.splice(0));
 		};
-		for (let i = 0; i < actions.length; i++) {
-			const action = actions[i];
-			const type = typeof action.type === "string" ? action.type : "";
-			if (type === "screenshot") {
+		for (const action of actions) switch (action.type) {
+			case "screenshot":
 				await flush();
 				result.readResults.push({
 					type: "screenshot",
 					...await this.screenshot()
 				});
-				continue;
-			}
-			if (type === "url") {
+				break;
+			case "url":
 				await flush();
 				result.readResults.push({
 					type: "url",
 					url: await this.currentUrl()
 				});
-				continue;
-			}
-			if (type === "cursor_position") {
+				break;
+			case "cursor_position":
 				await flush();
-				const pos = await this.currentMousePosition();
 				result.readResults.push({
 					type: "cursor_position",
-					...pos
+					...await this.currentMousePosition()
 				});
-				continue;
-			}
-			if (type === "goto") {
-				const url = normalizeGotoUrl(action.url) ?? "";
+				break;
+			case "goto":
 				pending.push(keypress(["Control", "l"]), {
 					type: "type_text",
-					type_text: { text: url }
+					type_text: { text: normalizeGotoUrl(action.url) ?? "" }
 				}, keypress(["Enter"]));
-				continue;
-			}
-			if (type === "back") {
+				break;
+			case "back":
 				pending.push(keypress(["Alt", "Left"]));
-				continue;
-			}
-			if (type === "forward") {
+				break;
+			case "forward":
 				pending.push(keypress(["Alt", "Right"]));
-				continue;
-			}
-			pending.push(toSdkAction(type, action, this.coordinateSystem, this.viewport));
+				break;
+			default:
+				pending.push(this.toSdkAction(action));
+				break;
 		}
 		await flush();
 		return result;
 	}
-	async runKernelBatch(actions) {
-		await this.client.browsers.computer.batch(this.sessionId, { actions });
-	}
-};
-function toSdkAction(type, action, coordinateSystem, viewport) {
-	switch (type) {
-		case "click": {
-			const clickHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "click_mouse",
-				click_mouse: {
-					x: point.x,
-					y: point.y,
-					button: clickMouseButtonOr(action.button, "left"),
-					...clickHoldKeys.length > 0 ? { hold_keys: clickHoldKeys } : {}
-				}
-			};
-		}
-		case "double_click": {
-			const doubleClickHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "click_mouse",
-				click_mouse: {
-					x: point.x,
-					y: point.y,
-					num_clicks: 2,
-					...doubleClickHoldKeys.length > 0 ? { hold_keys: doubleClickHoldKeys } : {}
-				}
-			};
-		}
-		case "mouse_down":
-		case "mouse_up": {
-			const mouseHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "click_mouse",
-				click_mouse: {
-					x: point.x,
-					y: point.y,
-					button: clickMouseButtonOr(action.button, "left"),
-					click_type: type === "mouse_down" ? "down" : "up",
-					...mouseHoldKeys.length > 0 ? { hold_keys: mouseHoldKeys } : {}
-				}
-			};
+	toSdkAction(action) {
+		switch (action.type) {
+			case "click": return this.clickAction(action, { button: mouseButton(action.button) });
+			case "double_click": return this.clickAction(action, { num_clicks: 2 });
+			case "mouse_down": return this.clickAction(action, {
+				button: mouseButton(action.button),
+				click_type: "down"
+			});
+			case "mouse_up": return this.clickAction(action, {
+				button: mouseButton(action.button),
+				click_type: "up"
+			});
+			case "type": return typeText(action);
+			case "keypress": return keypress(action.keys, action.duration);
+			case "scroll": return this.scrollAction(action);
+			case "move": return this.moveAction(action);
+			case "drag": return this.dragAction(action);
+			case "wait": return waitAction(action);
+			default: return unreachable(action);
 		}
-		case "type": return {
-			type: "type_text",
-			type_text: { text: typeof action.text === "string" ? action.text : "" }
+	}
+	clickAction(action, extra) {
+		const point = this.toViewportPoint(action.x, action.y);
+		return {
+			type: "click_mouse",
+			click_mouse: {
+				x: point.x,
+				y: point.y,
+				...extra,
+				...holdKeys(action.hold_keys)
+			}
 		};
-		case "keypress": return keypress(toStringArray(action.keys), action.duration);
-		case "scroll": {
-			const scrollHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "scroll",
-				scroll: {
-					x: point.x,
-					y: point.y,
-					delta_x: toInt(action.scroll_x),
-					delta_y: toInt(action.scroll_y),
-					...scrollHoldKeys.length > 0 ? { hold_keys: scrollHoldKeys } : {}
-				}
-			};
-		}
-		case "move": {
-			const moveHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "move_mouse",
-				move_mouse: {
-					x: point.x,
-					y: point.y,
-					...moveHoldKeys.length > 0 ? { hold_keys: moveHoldKeys } : {}
-				}
-			};
-		}
-		case "drag": {
-			const dragHoldKeys = readHoldKeys(action.hold_keys);
-			return {
-				type: "drag_mouse",
-				drag_mouse: {
-					path: toPath(action.path, coordinateSystem, viewport),
-					button: dragMouseButtonOr(action.button, "left"),
-					...dragHoldKeys.length > 0 ? { hold_keys: dragHoldKeys } : {}
-				}
-			};
-		}
-		case "wait": return {
-			type: "sleep",
-			sleep: { duration_ms: typeof action.ms === "number" ? Math.trunc(action.ms) : 1e3 }
+	}
+	scrollAction(action) {
+		const point = this.toViewportPoint(action.x ?? 0, action.y ?? 0);
+		return {
+			type: "scroll",
+			scroll: {
+				x: point.x,
+				y: point.y,
+				delta_x: Math.trunc(action.scroll_x ?? 0),
+				delta_y: Math.trunc(action.scroll_y ?? 0),
+				...holdKeys(action.hold_keys)
+			}
 		};
-		default: throw new Error(`unknown computer action type: ${type}`);
 	}
-}
-function toInt(value) {
-	if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value);
-	if (typeof value === "string" && value.trim()) {
-		const n = Number(value);
-		if (Number.isFinite(n)) return Math.trunc(n);
+	moveAction(action) {
+		const point = this.toViewportPoint(action.x, action.y);
+		return {
+			type: "move_mouse",
+			move_mouse: {
+				x: point.x,
+				y: point.y
+			}
+		};
 	}
-	return 0;
-}
-function stringOr(value, fallback) {
-	return typeof value === "string" && value.length > 0 ? value : fallback;
+	dragAction(action) {
+		return {
+			type: "drag_mouse",
+			drag_mouse: {
+				path: action.path.map((point) => {
+					const transformed = this.toViewportPoint(point.x, point.y);
+					return [transformed.x, transformed.y];
+				}),
+				button: dragButton(action.button),
+				...holdKeys(action.hold_keys)
+			}
+		};
+	}
+	toViewportPoint(x, y) {
+		if (this.coordinateSystem.type === "pixel") return {
+			x: Math.trunc(x),
+			y: Math.trunc(y)
+		};
+		const [min, max] = this.coordinateSystem.range;
+		const scale = max - min;
+		if (scale <= 0) return {
+			x: Math.trunc(x),
+			y: Math.trunc(y)
+		};
+		return {
+			x: clamp(Math.round((x - min) / scale * this.viewport.width), 0, this.viewport.width - 1),
+			y: clamp(Math.round((y - min) / scale * this.viewport.height), 0, this.viewport.height - 1)
+		};
+	}
+	async runKernelBatch(actions) {
+		await this.client.browsers.computer.batch(this.sessionId, { actions });
+	}
+};
+const CLICK_BUTTONS = new Set([
+	"left",
+	"right",
+	"middle",
+	"back",
+	"forward"
+]);
+const DRAG_BUTTONS = new Set([
+	"left",
+	"right",
+	"middle"
+]);
+function mouseButton(value) {
+	return value !== void 0 && CLICK_BUTTONS.has(value) ? value : "left";
 }
-function clickMouseButtonOr(value, fallback) {
-	const candidate = stringOr(value, fallback);
-	if (candidate === "left" || candidate === "right" || candidate === "middle" || candidate === "back" || candidate === "forward") return candidate;
-	return fallback;
+function dragButton(value) {
+	return value !== void 0 && DRAG_BUTTONS.has(value) ? value : "left";
 }
-function dragMouseButtonOr(value, fallback) {
-	const candidate = stringOr(value, fallback);
-	if (candidate === "left" || candidate === "right" || candidate === "middle") return candidate;
-	return fallback;
+function typeText(action) {
+	return {
+		type: "type_text",
+		type_text: { text: action.text }
+	};
 }
-function toStringArray(value) {
-	return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
+function waitAction(action) {
+	return {
+		type: "sleep",
+		sleep: { duration_ms: Math.trunc(action.ms ?? 1e3) }
+	};
 }
-function readHoldKeys(value) {
-	return toStringArray(value).map(normalizeKernelKey);
+function holdKeys(keys) {
+	if (!keys || keys.length === 0) return {};
+	return { hold_keys: keys.map(normalizeKernelKey) };
 }
-function keypress(keys, duration = void 0) {
+function keypress(keys, duration) {
 	const translated = keys.flatMap(normalizeKernelKeyCombo);
 	const pressedKeys = translated.filter((key) => !isKernelModifierKey(key));
-	const holdKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1);
+	const heldKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1);
 	return {
 		type: "press_key",
 		press_key: {
 			keys: pressedKeys.length > 0 ? pressedKeys : translated.slice(-1),
-			...holdKeys.length > 0 ? { hold_keys: holdKeys } : {},
+			...heldKeys.length > 0 ? { hold_keys: heldKeys } : {},
 			...typeof duration === "number" && Number.isFinite(duration) && duration > 0 ? { duration: Math.trunc(duration) } : {}
 		}
 	};
 }
-function toPath(value, coordinateSystem = { type: "pixel" }, viewport = {
-	width: 1920,
-	height: 1080
-}) {
-	if (!Array.isArray(value)) return [];
-	return value.map((point) => toPathPoint(point, coordinateSystem, viewport));
-}
-function toPathPoint(value, coordinateSystem, viewport) {
-	if (Array.isArray(value)) {
-		const point = transformPoint(toInt(value[0]), toInt(value[1]), coordinateSystem, viewport);
-		return [point.x, point.y];
-	}
-	if (value && typeof value === "object") {
-		const point = value;
-		const transformed = transformPoint(toInt(point.x), toInt(point.y), coordinateSystem, viewport);
-		return [transformed.x, transformed.y];
-	}
-	return [0, 0];
-}
-function toViewportPoint(action, coordinateSystem, viewport) {
-	return transformPoint(toInt(action.x), toInt(action.y), coordinateSystem, viewport);
-}
-function transformPoint(x, y, coordinateSystem, viewport) {
-	if (coordinateSystem.type === "pixel") return {
-		x,
-		y
-	};
-	const [min, max] = coordinateSystem.range;
-	const scale = max - min;
-	if (scale <= 0) return {
-		x,
-		y
-	};
-	return {
-		x: clamp(Math.round((x - min) / scale * viewport.width), 0, viewport.width - 1),
-		y: clamp(Math.round((y - min) / scale * viewport.height), 0, viewport.height - 1)
-	};
-}
 function clamp(value, min, max) {
 	return Math.max(min, Math.min(max, value));
 }
+function unreachable(action) {
+	throw new Error(`unknown computer action type: ${JSON.stringify(action)}`);
+}
 //#endregion
 //#region src/tools.ts
 function createCuaComputerTools(args) {
-	const translator = new InternalComputerTranslator(args);
+	return buildCuaComputerTools(args, new InternalComputerTranslator(args));
+}
+/** Build executor tools against an existing translator (internal; not part of the package surface). */
+function buildCuaComputerTools(args, translator) {
 	return withNavigationTool(args).map((executor) => createExecutorTool(executor, translator));
 }
 function withNavigationTool(args) {
@@ -502,10 +462,11 @@ async function executeNavigationTool(translator, params) {
 		if (action === "url") {
 			url = await translator.currentUrl();
 			statusText = `Current URL: ${url}`;
-		} else await translator.executeBatch([{
-			type: action,
-			url: params.url
+		} else if (action === "goto") await translator.executeBatch([{
+			type: "goto",
+			url: params.url ?? ""
 		}]);
+		else await translator.executeBatch([{ type: action }]);
 		const screenshot = await translator.screenshot();
 		return {
 			content: [{
@@ -561,14 +522,10 @@ var CuaRuntimeController = class {
 		this.translator = this.createTranslator();
 	}
 	tools() {
-		return [...createCuaComputerTools({
-			browser: this.options.browser,
-			client: this.options.client,
+		return [...buildCuaComputerTools({
 			toolExecutors: this.runtimeSpec.toolExecutors,
-			coordinateSystem: this.runtimeSpec.coordinateSystem,
-			screenshot: this.runtimeSpec.screenshot,
 			computerUseExtra: this.options.computerUseExtra
-		}), ...this.options.extraTools ?? []];
+		}, this.translator), ...this.options.extraTools ?? []];
 	}
 	onPayload() {
 		const runtimeSpec = this.runtimeSpec;
@@ -605,7 +562,9 @@ async function getCuaEnvApiKeyAndHeaders(model) {
 var CuaAgent = class extends Agent {
 	runtime;
 	ownsSystemPrompt;
+	runtimeDirty = false;
 	stateProxy;
+	stateProxyTarget;
 	constructor(options) {
 		const { browser, client, initialState, onPayload, streamFn, prepareNextTurn, extraTools, computerUseExtra, ...agentOptions } = options;
 		const runtime = new CuaRuntimeController({
@@ -638,13 +597,17 @@ var CuaAgent = class extends Agent {
 		this.runtime = runtime;
 		this.ownsSystemPrompt = initialState.systemPrompt === void 0;
 		/**
-		* pi calls `prepareNextTurn` between provider requests. Wrapping it lets CUA
-		* honor any user-provided turn update while also refreshing provider-specific
-		* defaults if that update changes the model.
+		* pi's loop only re-reads model/tools/prompt between provider requests
+		* through `prepareNextTurn`. The wrapper stays pass-through (returning
+		* `undefined`, i.e. stock pi behavior) until either the user hook returns
+		* an update or a mid-run model assignment marks the CUA runtime dirty —
+		* only then is a turn update built from current state.
 		*/
 		this.prepareNextTurn = async (signal) => {
 			const update = await prepareNextTurn?.(signal);
 			if (update?.model) this.applyRuntime(update.model);
+			if (!update && !this.runtimeDirty) return void 0;
+			this.runtimeDirty = false;
 			const state = super.state;
 			const context = update?.context ?? {
 				systemPrompt: state.systemPrompt,
@@ -668,17 +631,22 @@ var CuaAgent = class extends Agent {
 	* and payload hooks for the selected provider.
 	*/
 	get state() {
-		if (!this.stateProxy) this.stateProxy = new Proxy(super.state, { set: (target, prop, value, receiver) => {
-			if (prop === "model") {
-				this.applyRuntime(value);
-				return true;
-			}
-			return Reflect.set(target, prop, value, receiver);
-		} });
+		const target = super.state;
+		if (!this.stateProxy || this.stateProxyTarget !== target) {
+			this.stateProxyTarget = target;
+			this.stateProxy = new Proxy(target, { set: (proxied, prop, value, receiver) => {
+				if (prop === "model") {
+					this.applyRuntime(value);
+					return true;
+				}
+				return Reflect.set(proxied, prop, value, receiver);
+			} });
+		}
 		return this.stateProxy;
 	}
 	applyRuntime(model) {
 		this.runtime.setModel(model);
+		this.runtimeDirty = true;
 		const state = super.state;
 		state.model = this.runtime.model;
 		state.tools = this.runtime.tools();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onkernel/cua-agent",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "Kernel browser computer-use Agent and AgentHarness classes built on pi-agent-core",
   "license": "MIT",
   "type": "module",
@@ -42,7 +42,7 @@
   "dependencies": {
     "@earendil-works/pi-agent-core": "0.79.1",
     "@earendil-works/pi-ai": "0.79.1",
-    "@onkernel/cua-ai": "0.2.2",
+    "@onkernel/cua-ai": "0.3.0",
     "@onkernel/sdk": "0.49.0",
     "sharp": "^0.34.5"
   },