@onkernel/cua-agent 0.3.2 → 0.3.4

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.3.4 - 2026-06-23
+
+- Add an opt-in `playwright` option to `CuaAgent` and `CuaAgentHarness` that
+  exposes a `playwright_execute` tool, running Playwright/TypeScript against
+  the live browser session via the Kernel SDK. Results, stdout, and stderr
+  come back as tool content; SDK-reported failures surface as content rather
+  than throwing. Adds the `PlaywrightDetails` export.
+
+## 0.3.3 - 2026-06-12
+
+- The action translator now consumes the canonical `CuaAction` union with an
+  exhaustive switch. Malformed action shapes fail loudly instead of silently
+  coercing (previously e.g. a click at 0,0); the documented mouse-button
+  coercion to `"left"` is unchanged.
+- `prepareNextTurn` no longer rebuilds the turn context on every turn: it
+  keeps stock pi behavior until a user hook returns an update or a mid-run
+  model assignment requires a refresh.
+- One translator instance per runtime is shared between the executor tools
+  and the provider screenshot capability.
+- The `CuaAgentHarness` README quickstart showcases session-backed turns and
+  mid-session model switching; `computerUseExtra` is documented with its
+  rationale.
+- Update the `@onkernel/cua-ai` dependency to 0.3.0.
+
 ## 0.3.2 - 2026-06-11
 
 - Update the `@onkernel/cua-ai` dependency to 0.2.2.

package/README.md

@@ -37,11 +37,17 @@ await agent.prompt("Open news.ycombinator.com and summarize the top story.");
 
 ## Quick Start (`CuaAgentHarness`)
 
+`prompt()` returns the turn's final assistant message, and every turn is
+persisted to the session — later prompts see the full transcript. Runtime
+config like the model can change between turns (or even mid-turn, applying at
+the next provider request):
+
 ```ts
 import { CuaAgentHarness, InMemorySessionRepo, NodeExecutionEnv } from "@onkernel/cua-agent";
+import type { AssistantMessage } from "@onkernel/cua-ai";
 
 const sessionRepo = new InMemorySessionRepo();
-const session = await sessionRepo.create({ id: "example" });
+const session = await sessionRepo.create({ id: "research" });
 
 const harness = new CuaAgentHarness({
   browser,
@@ -51,22 +57,26 @@ const harness = new CuaAgentHarness({
   session,
 });
 
-const response = await harness.prompt("Open example.com and tell me the current URL.");
-const branch = await session.getBranch();
-const lastAssistant = [...branch]
-  .reverse()
-  .flatMap((entry) =>
-    entry.type === "message" && entry.message.role === "assistant" ? [entry.message] : [],
-  )[0];
-const assistant = lastAssistant ?? response;
-const assistantText = assistant.content
-  .flatMap((block) => (block.type === "text" ? [block.text] : []))
-  .join("")
-  .trim();
-console.log("assistant stopReason:", assistant.stopReason);
-console.log("assistant text:", assistantText || "(no text)");
+const textOf = (message: AssistantMessage) =>
+  message.content.flatMap((block) => (block.type === "text" ? [block.text] : [])).join("").trim();
+
+// Turn 1: a session-backed prompt.
+const first = await harness.prompt("Open example.com and describe what you see.");
+console.log(textOf(first));
+
+// Swap providers mid-session; CUA tools and the default prompt refresh to match.
+await harness.setModel("anthropic:claude-opus-4-7");
+
+// Turn 2 continues the same transcript on the new model.
+const second = await harness.prompt("Open the most relevant link from what you found.");
+console.log(textOf(second));
 ```
 
+While a turn is running, `steer()` injects course corrections, `followUp()`
+queues the next instruction, and `subscribe()` streams the underlying agent
+events. `compact()` and session branching are available for long-running
+transcripts — see the pi-agent-core docs for the full harness lifecycle.
+
 Use `CuaAgent` when you want direct pi `Agent` control: raw message state,
 lifecycle events, custom streaming, and explicit prompt/continue/queue control.
 Reach for the harness shape when you want an app layer around the loop:
@@ -88,6 +98,8 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions:
 - CUA model refs (`"provider:model"`) accepted where pi expects a concrete model
 - `extraTools` to add your own pi tools alongside the built-in browser tools
 - `computerUseExtra: true` to let the model use a small navigation helper
+- `playwright: true` to let the model run Playwright/TypeScript against the
+  live browser session
 
 If auth callbacks are omitted, both classes default to CUA env var conventions:
 - OpenAI: `OPENAI_API_KEY`
@@ -108,9 +120,24 @@ computer-use tools. This is useful when the model needs to call
 application-specific code, such as looking up a record, writing a database row,
 or handing off to another service while it also controls the browser.
 
-`computerUseExtra: true` adds the `computer_use_extra` tool. Use it when you
-want one compact helper for common browser navigation/read operations:
-`goto`, `back`, `forward`, and `url`.
+Not every provider's native computer-use vocabulary includes browser
+navigation — some models can click and type but have no direct way to open a
+URL or go back. `computerUseExtra: true` adds `computer_use_extra`, a
+provider-neutral escape hatch exposing `goto`, `back`, `forward`, and `url`
+so navigation works uniformly regardless of which model is driving.
+
+Some steps are awkward as raw pointer/keyboard actions: precise DOM reads,
+form fills, data extraction, or waiting on a specific selector.
+`playwright: true` adds `playwright_execute`, which runs Playwright/TypeScript
+directly against the live browser session. `page`, `context`, and `browser`
+are in scope and the code may `return` a JSON-serializable value. Each call
+runs in a fresh JS context (locals don't persist across calls) but the
+browser session does carry over. No screenshot is returned automatically;
+request one on a follow-up turn when the model needs to see the page.
+Playwright-level failures come back as tool content (so the model can adapt)
+rather than thrown errors. Verified e2e
+against Anthropic, Tzafon, and Yutori CUA models; OpenAI and Google are
+unit-tested.
 
 ### Model Switching
 

package/dist/index.d.ts

@@ -16,6 +16,7 @@ interface ComputerToolOptions {
   coordinateSystem?: ComputerToolCoordinateSystem;
   screenshot?: CuaScreenshotSpec;
   computerUseExtra?: boolean;
+  playwright?: boolean;
 }
 interface BatchDetails {
   statusText: string;
@@ -36,10 +37,34 @@ interface NavigationDetails {
   statusText: string;
   url?: string;
 }
+/**
+ * Structured details for a `playwright_execute` tool result. Library
+ * consumers can read these directly instead of re-parsing the model-facing
+ * tool content blocks.
+ *
+ * - `success` — whether the Playwright code itself completed without error.
+ *   A `false` value means the code threw or the SDK reported failure; in
+ *   that case the failure is also surfaced as tool content for the model.
+ * - `statusText` — short human-readable status (success or failure summary).
+ * - `result` — present only when the code returned a JSON-serializable value.
+ * - `stdout`/`stderr` — raw daemon output, present whenever the daemon
+ *   reported a non-empty value on that stream (may be whitespace-only).
+ * - `error` — present only when `success` is `false`; the error message from
+ *   the daemon.
+ */
+interface PlaywrightDetails {
+  success: boolean;
+  statusText: string;
+  result?: unknown;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+}
 type BatchTool = AgentTool<TSchema, BatchDetails>;
 type NavigationTool = AgentTool<TSchema, NavigationDetails>;
+type PlaywrightTool = AgentTool<TSchema, PlaywrightDetails>;
 type ActionTool = AgentTool<TSchema, BatchDetails>;
-type CuaExecutorTool = BatchTool | NavigationTool | ActionTool;
+type CuaExecutorTool = BatchTool | NavigationTool | PlaywrightTool | ActionTool;
 declare function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTool[];
 //#endregion
 //#region src/agent.d.ts
@@ -74,7 +99,8 @@ type CuaAgentOptions = Omit<AgentOptions, "initialState"> & {
   client: Kernel; /** Initial pi state plus a CUA-aware model value. */
   initialState: CuaAgentInitialState; /** Add your own pi tools alongside the built-in browser tools. */
   extraTools?: AgentTool[]; /** Expose a helper for browser navigation and URL reads. */
-  computerUseExtra?: boolean;
+  computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */
+  playwright?: boolean;
 };
 /**
  * Constructor options for {@link CuaAgentHarness}.
@@ -88,7 +114,8 @@ type CuaAgentHarnessOptions<TSkill extends Skill = Skill, TPromptTemplate extend
   client: Kernel; /** Model used by the harness. CUA refs are resolved before pi sees the model. */
   model: CuaRuntimeInput; /** Add your own pi tools alongside the built-in browser tools. */
   extraTools?: AgentTool[]; /** Expose a helper for browser navigation and URL reads. */
-  computerUseExtra?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */
+  computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */
+  playwright?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */
   onPayload?: SimpleStreamOptions["onPayload"];
 };
 /**
@@ -102,7 +129,9 @@ type CuaAgentHarnessOptions<TSkill extends Skill = Skill, TPromptTemplate extend
 declare class CuaAgent extends Agent {
   private readonly runtime;
   private readonly ownsSystemPrompt;
+  private runtimeDirty;
   private stateProxy?;
+  private stateProxyTarget?;
   constructor(options: CuaAgentOptions);
   /**
    * Return a state proxy so `agent.state.model = "provider:model"` can behave
@@ -135,4 +164,4 @@ declare class CuaAgentHarness<TSkill extends Skill = Skill, TPromptTemplate exte
   setActiveTools(toolNames: string[]): Promise<void>;
 }
 //#endregion
-export { type BatchDetails, type ComputerToolOptions, CuaAgent, CuaAgentHarness, type CuaAgentHarnessOptions, type CuaAgentOptions, type CuaAgentState, type CuaExecutorTool, type KernelBrowser, type NavigationDetails, NodeExecutionEnv, createCuaComputerTools };
+export { type BatchDetails, type ComputerToolOptions, CuaAgent, CuaAgentHarness, type CuaAgentHarnessOptions, type CuaAgentOptions, type CuaAgentState, type CuaExecutorTool, type KernelBrowser, type NavigationDetails, NodeExecutionEnv, type PlaywrightDetails, createCuaComputerTools };

package/dist/index.js

@@ -1,6 +1,6 @@
 import { Agent, AgentHarness } from "@earendil-works/pi-agent-core";
 import { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node";
-import { CUA_NAVIGATION_TOOL_NAME, createCuaNavigationToolDefinition, getCuaEnvApiKey, normalizeGotoUrl, resolveCuaRuntimeSpec, streamSimple } from "@onkernel/cua-ai";
+import { CUA_NAVIGATION_TOOL_NAME, CUA_PLAYWRIGHT_TOOL_NAME, createCuaNavigationToolDefinition, createCuaPlaywrightToolDefinition, getCuaEnvApiKey, normalizeGotoUrl, resolveCuaRuntimeSpec, streamSimple } from "@onkernel/cua-ai";
 import sharp from "sharp";
 export * from "@earendil-works/pi-agent-core";
 //#region src/translator/keys.ts
@@ -156,10 +156,18 @@ var InternalComputerTranslator = class {
 	async currentMousePosition() {
 		const pos = await this.client.browsers.computer.getMousePosition(this.sessionId);
 		return {
-			x: toInt(pos.x),
-			y: toInt(pos.y)
+			x: Math.trunc(pos.x),
+			y: Math.trunc(pos.y)
 		};
 	}
+	async executePlaywright(code, timeoutSec) {
+		const truncated = timeoutSec !== void 0 ? Math.trunc(timeoutSec) : void 0;
+		const timeout = truncated !== void 0 && truncated >= 1 ? Math.min(truncated, PLAYWRIGHT_MAX_TIMEOUT_SEC) : void 0;
+		return this.client.browsers.playwright.execute(this.sessionId, {
+			code,
+			...timeout !== void 0 ? { timeout_sec: timeout } : {}
+		});
+	}
 	async executeBatch(actions) {
 		const result = { readResults: [] };
 		const pending = [];
@@ -167,248 +175,210 @@ var InternalComputerTranslator = class {
 			if (pending.length === 0) return;
 			await this.runKernelBatch(pending.splice(0));
 		};
-		for (let i = 0; i < actions.length; i++) {
-			const action = actions[i];
-			const type = typeof action.type === "string" ? action.type : "";
-			if (type === "screenshot") {
+		for (const action of actions) switch (action.type) {
+			case "screenshot":
 				await flush();
 				result.readResults.push({
 					type: "screenshot",
 					...await this.screenshot()
 				});
-				continue;
-			}
-			if (type === "url") {
+				break;
+			case "url":
 				await flush();
 				result.readResults.push({
 					type: "url",
 					url: await this.currentUrl()
 				});
-				continue;
-			}
-			if (type === "cursor_position") {
+				break;
+			case "cursor_position":
 				await flush();
-				const pos = await this.currentMousePosition();
 				result.readResults.push({
 					type: "cursor_position",
-					...pos
+					...await this.currentMousePosition()
 				});
-				continue;
-			}
-			if (type === "goto") {
-				const url = normalizeGotoUrl(action.url) ?? "";
+				break;
+			case "goto":
 				pending.push(keypress(["Control", "l"]), {
 					type: "type_text",
-					type_text: { text: url }
+					type_text: { text: normalizeGotoUrl(action.url) ?? "" }
 				}, keypress(["Enter"]));
-				continue;
-			}
-			if (type === "back") {
+				break;
+			case "back":
 				pending.push(keypress(["Alt", "Left"]));
-				continue;
-			}
-			if (type === "forward") {
+				break;
+			case "forward":
 				pending.push(keypress(["Alt", "Right"]));
-				continue;
-			}
-			pending.push(toSdkAction(type, action, this.coordinateSystem, this.viewport));
+				break;
+			default:
+				pending.push(this.toSdkAction(action));
+				break;
 		}
 		await flush();
 		return result;
 	}
-	async runKernelBatch(actions) {
-		await this.client.browsers.computer.batch(this.sessionId, { actions });
-	}
-};
-function toSdkAction(type, action, coordinateSystem, viewport) {
-	switch (type) {
-		case "click": {
-			const clickHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "click_mouse",
-				click_mouse: {
-					x: point.x,
-					y: point.y,
-					button: clickMouseButtonOr(action.button, "left"),
-					...clickHoldKeys.length > 0 ? { hold_keys: clickHoldKeys } : {}
-				}
-			};
-		}
-		case "double_click": {
-			const doubleClickHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "click_mouse",
-				click_mouse: {
-					x: point.x,
-					y: point.y,
-					num_clicks: 2,
-					...doubleClickHoldKeys.length > 0 ? { hold_keys: doubleClickHoldKeys } : {}
-				}
-			};
-		}
-		case "mouse_down":
-		case "mouse_up": {
-			const mouseHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "click_mouse",
-				click_mouse: {
-					x: point.x,
-					y: point.y,
-					button: clickMouseButtonOr(action.button, "left"),
-					click_type: type === "mouse_down" ? "down" : "up",
-					...mouseHoldKeys.length > 0 ? { hold_keys: mouseHoldKeys } : {}
-				}
-			};
+	toSdkAction(action) {
+		switch (action.type) {
+			case "click": return this.clickAction(action, { button: mouseButton(action.button) });
+			case "double_click": return this.clickAction(action, { num_clicks: 2 });
+			case "mouse_down": return this.clickAction(action, {
+				button: mouseButton(action.button),
+				click_type: "down"
+			});
+			case "mouse_up": return this.clickAction(action, {
+				button: mouseButton(action.button),
+				click_type: "up"
+			});
+			case "type": return typeText(action);
+			case "keypress": return keypress(action.keys, action.duration);
+			case "scroll": return this.scrollAction(action);
+			case "move": return this.moveAction(action);
+			case "drag": return this.dragAction(action);
+			case "wait": return waitAction(action);
+			default: return unreachable(action);
 		}
-		case "type": return {
-			type: "type_text",
-			type_text: { text: typeof action.text === "string" ? action.text : "" }
+	}
+	clickAction(action, extra) {
+		const point = this.toViewportPoint(action.x, action.y);
+		return {
+			type: "click_mouse",
+			click_mouse: {
+				x: point.x,
+				y: point.y,
+				...extra,
+				...holdKeys(action.hold_keys)
+			}
 		};
-		case "keypress": return keypress(toStringArray(action.keys), action.duration);
-		case "scroll": {
-			const scrollHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "scroll",
-				scroll: {
-					x: point.x,
-					y: point.y,
-					delta_x: toInt(action.scroll_x),
-					delta_y: toInt(action.scroll_y),
-					...scrollHoldKeys.length > 0 ? { hold_keys: scrollHoldKeys } : {}
-				}
-			};
-		}
-		case "move": {
-			const moveHoldKeys = readHoldKeys(action.hold_keys);
-			const point = toViewportPoint(action, coordinateSystem, viewport);
-			return {
-				type: "move_mouse",
-				move_mouse: {
-					x: point.x,
-					y: point.y,
-					...moveHoldKeys.length > 0 ? { hold_keys: moveHoldKeys } : {}
-				}
-			};
-		}
-		case "drag": {
-			const dragHoldKeys = readHoldKeys(action.hold_keys);
-			return {
-				type: "drag_mouse",
-				drag_mouse: {
-					path: toPath(action.path, coordinateSystem, viewport),
-					button: dragMouseButtonOr(action.button, "left"),
-					...dragHoldKeys.length > 0 ? { hold_keys: dragHoldKeys } : {}
-				}
-			};
-		}
-		case "wait": return {
-			type: "sleep",
-			sleep: { duration_ms: typeof action.ms === "number" ? Math.trunc(action.ms) : 1e3 }
+	}
+	scrollAction(action) {
+		const point = this.toViewportPoint(action.x ?? 0, action.y ?? 0);
+		return {
+			type: "scroll",
+			scroll: {
+				x: point.x,
+				y: point.y,
+				delta_x: Math.trunc(action.scroll_x ?? 0),
+				delta_y: Math.trunc(action.scroll_y ?? 0),
+				...holdKeys(action.hold_keys)
+			}
 		};
-		default: throw new Error(`unknown computer action type: ${type}`);
 	}
-}
-function toInt(value) {
-	if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value);
-	if (typeof value === "string" && value.trim()) {
-		const n = Number(value);
-		if (Number.isFinite(n)) return Math.trunc(n);
+	moveAction(action) {
+		const point = this.toViewportPoint(action.x, action.y);
+		return {
+			type: "move_mouse",
+			move_mouse: {
+				x: point.x,
+				y: point.y
+			}
+		};
 	}
-	return 0;
-}
-function stringOr(value, fallback) {
-	return typeof value === "string" && value.length > 0 ? value : fallback;
+	dragAction(action) {
+		return {
+			type: "drag_mouse",
+			drag_mouse: {
+				path: action.path.map((point) => {
+					const transformed = this.toViewportPoint(point.x, point.y);
+					return [transformed.x, transformed.y];
+				}),
+				button: dragButton(action.button),
+				...holdKeys(action.hold_keys)
+			}
+		};
+	}
+	toViewportPoint(x, y) {
+		if (this.coordinateSystem.type === "pixel") return {
+			x: Math.trunc(x),
+			y: Math.trunc(y)
+		};
+		const [min, max] = this.coordinateSystem.range;
+		const scale = max - min;
+		if (scale <= 0) return {
+			x: Math.trunc(x),
+			y: Math.trunc(y)
+		};
+		return {
+			x: clamp(Math.round((x - min) / scale * this.viewport.width), 0, this.viewport.width - 1),
+			y: clamp(Math.round((y - min) / scale * this.viewport.height), 0, this.viewport.height - 1)
+		};
+	}
+	async runKernelBatch(actions) {
+		await this.client.browsers.computer.batch(this.sessionId, { actions });
+	}
+};
+const PLAYWRIGHT_MAX_TIMEOUT_SEC = 300;
+const CLICK_BUTTONS = new Set([
+	"left",
+	"right",
+	"middle",
+	"back",
+	"forward"
+]);
+const DRAG_BUTTONS = new Set([
+	"left",
+	"right",
+	"middle"
+]);
+function mouseButton(value) {
+	return value !== void 0 && CLICK_BUTTONS.has(value) ? value : "left";
 }
-function clickMouseButtonOr(value, fallback) {
-	const candidate = stringOr(value, fallback);
-	if (candidate === "left" || candidate === "right" || candidate === "middle" || candidate === "back" || candidate === "forward") return candidate;
-	return fallback;
+function dragButton(value) {
+	return value !== void 0 && DRAG_BUTTONS.has(value) ? value : "left";
 }
-function dragMouseButtonOr(value, fallback) {
-	const candidate = stringOr(value, fallback);
-	if (candidate === "left" || candidate === "right" || candidate === "middle") return candidate;
-	return fallback;
+function typeText(action) {
+	return {
+		type: "type_text",
+		type_text: { text: action.text }
+	};
 }
-function toStringArray(value) {
-	return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
+function waitAction(action) {
+	return {
+		type: "sleep",
+		sleep: { duration_ms: Math.trunc(action.ms ?? 1e3) }
+	};
 }
-function readHoldKeys(value) {
-	return toStringArray(value).map(normalizeKernelKey);
+function holdKeys(keys) {
+	if (!keys || keys.length === 0) return {};
+	return { hold_keys: keys.map(normalizeKernelKey) };
 }
-function keypress(keys, duration = void 0) {
+function keypress(keys, duration) {
 	const translated = keys.flatMap(normalizeKernelKeyCombo);
 	const pressedKeys = translated.filter((key) => !isKernelModifierKey(key));
-	const holdKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1);
+	const heldKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1);
 	return {
 		type: "press_key",
 		press_key: {
 			keys: pressedKeys.length > 0 ? pressedKeys : translated.slice(-1),
-			...holdKeys.length > 0 ? { hold_keys: holdKeys } : {},
+			...heldKeys.length > 0 ? { hold_keys: heldKeys } : {},
 			...typeof duration === "number" && Number.isFinite(duration) && duration > 0 ? { duration: Math.trunc(duration) } : {}
 		}
 	};
 }
-function toPath(value, coordinateSystem = { type: "pixel" }, viewport = {
-	width: 1920,
-	height: 1080
-}) {
-	if (!Array.isArray(value)) return [];
-	return value.map((point) => toPathPoint(point, coordinateSystem, viewport));
-}
-function toPathPoint(value, coordinateSystem, viewport) {
-	if (Array.isArray(value)) {
-		const point = transformPoint(toInt(value[0]), toInt(value[1]), coordinateSystem, viewport);
-		return [point.x, point.y];
-	}
-	if (value && typeof value === "object") {
-		const point = value;
-		const transformed = transformPoint(toInt(point.x), toInt(point.y), coordinateSystem, viewport);
-		return [transformed.x, transformed.y];
-	}
-	return [0, 0];
-}
-function toViewportPoint(action, coordinateSystem, viewport) {
-	return transformPoint(toInt(action.x), toInt(action.y), coordinateSystem, viewport);
-}
-function transformPoint(x, y, coordinateSystem, viewport) {
-	if (coordinateSystem.type === "pixel") return {
-		x,
-		y
-	};
-	const [min, max] = coordinateSystem.range;
-	const scale = max - min;
-	if (scale <= 0) return {
-		x,
-		y
-	};
-	return {
-		x: clamp(Math.round((x - min) / scale * viewport.width), 0, viewport.width - 1),
-		y: clamp(Math.round((y - min) / scale * viewport.height), 0, viewport.height - 1)
-	};
-}
 function clamp(value, min, max) {
 	return Math.max(min, Math.min(max, value));
 }
+function unreachable(action) {
+	throw new Error(`unknown computer action type: ${JSON.stringify(action)}`);
+}
 //#endregion
 //#region src/tools.ts
 function createCuaComputerTools(args) {
-	const translator = new InternalComputerTranslator(args);
-	return withNavigationTool(args).map((executor) => createExecutorTool(executor, translator));
+	return buildCuaComputerTools(args, new InternalComputerTranslator(args));
+}
+/** Build executor tools against an existing translator (internal; not part of the package surface). */
+function buildCuaComputerTools(args, translator) {
+	return withExtraTools(args).map((executor) => createExecutorTool(executor, translator));
 }
-function withNavigationTool(args) {
+function withExtraTools(args) {
 	const executors = [...args.toolExecutors];
 	const existing = new Set(executors.map((executor) => executor.definition.name));
-	if (args.computerUseExtra && !existing.has(CUA_NAVIGATION_TOOL_NAME)) {
-		const definition = createCuaNavigationToolDefinition();
-		executors.push({
-			kind: "navigation",
-			definition
-		});
-	}
+	if (args.computerUseExtra && !existing.has(CUA_NAVIGATION_TOOL_NAME)) executors.push({
+		kind: "navigation",
+		definition: createCuaNavigationToolDefinition()
+	});
+	if (args.playwright && !existing.has(CUA_PLAYWRIGHT_TOOL_NAME)) executors.push({
+		kind: "playwright",
+		definition: createCuaPlaywrightToolDefinition()
+	});
 	return executors;
 }
 function createExecutorTool(executor, translator) {
@@ -422,6 +392,16 @@ function createExecutorTool(executor, translator) {
 			return executeNavigationTool(translator, asNavigationInput(params));
 		}
 	};
+	if (isPlaywrightExecutor(executor)) return {
+		name: definition.name,
+		label: definition.name,
+		description: definition.description,
+		parameters: definition.parameters,
+		executionMode: "sequential",
+		async execute(_toolCallId, params) {
+			return executePlaywrightTool(translator, asPlaywrightInput(params));
+		}
+	};
 	return {
 		name: definition.name,
 		label: definition.name,
@@ -436,6 +416,9 @@ function createExecutorTool(executor, translator) {
 function isNavigationExecutor(executor) {
 	return "kind" in executor && executor.kind === "navigation";
 }
+function isPlaywrightExecutor(executor) {
+	return "kind" in executor && executor.kind === "playwright";
+}
 async function executeBatchTool(translator, params) {
 	const content = [];
 	const readResults = [];
@@ -502,10 +485,11 @@ async function executeNavigationTool(translator, params) {
 		if (action === "url") {
 			url = await translator.currentUrl();
 			statusText = `Current URL: ${url}`;
-		} else await translator.executeBatch([{
-			type: action,
-			url: params.url
+		} else if (action === "goto") await translator.executeBatch([{
+			type: "goto",
+			url: params.url ?? ""
 		}]);
+		else await translator.executeBatch([{ type: action }]);
 		const screenshot = await translator.screenshot();
 		return {
 			content: [{
@@ -526,6 +510,50 @@ async function executeNavigationTool(translator, params) {
 		throw new Error(`${action} failed: ${errorMessage(err)}`, { cause: err });
 	}
 }
+async function executePlaywrightTool(translator, params) {
+	try {
+		const execution = await translator.executePlaywright(params.code, params.timeout_sec);
+		const content = [];
+		if (execution.result !== void 0) content.push({
+			type: "text",
+			text: `result: ${formatPlaywrightResult(execution.result)}`
+		});
+		if (execution.stdout?.trim()) content.push({
+			type: "text",
+			text: `stdout:\n${execution.stdout.trimEnd()}`
+		});
+		if (execution.stderr?.trim()) content.push({
+			type: "text",
+			text: `stderr:\n${execution.stderr.trimEnd()}`
+		});
+		if (!execution.success) content.push({
+			type: "text",
+			text: `error: ${execution.error ?? "playwright execution reported failure"}`
+		});
+		const statusText = execution.success ? "Playwright executed successfully." : `Playwright execution failed: ${execution.error ?? "unknown error"}`;
+		if (content.length === 0) content.push({
+			type: "text",
+			text: statusText
+		});
+		const details = {
+			success: execution.success,
+			statusText
+		};
+		if (execution.result !== void 0) details.result = execution.result;
+		if (execution.stdout) details.stdout = execution.stdout;
+		if (execution.stderr) details.stderr = execution.stderr;
+		if (execution.error) details.error = execution.error;
+		return {
+			content,
+			details
+		};
+	} catch (err) {
+		throw new Error(`playwright_execute failed: ${errorMessage(err)}`, { cause: err });
+	}
+}
+function formatPlaywrightResult(result) {
+	return typeof result === "string" ? result : JSON.stringify(result);
+}
 function errorMessage(err) {
 	return err instanceof Error ? err.message : String(err);
 }
@@ -533,6 +561,10 @@ function asNavigationInput(value) {
 	if (value && typeof value === "object" && typeof value.action === "string") return value;
 	throw new Error("invalid computer_use_extra parameters");
 }
+function asPlaywrightInput(value) {
+	if (value && typeof value === "object" && typeof value.code === "string") return value;
+	throw new Error("invalid playwright_execute parameters");
+}
 //#endregion
 //#region src/agent.ts
 /**
@@ -561,14 +593,11 @@ var CuaRuntimeController = class {
 		this.translator = this.createTranslator();
 	}
 	tools() {
-		return [...createCuaComputerTools({
-			browser: this.options.browser,
-			client: this.options.client,
+		return [...buildCuaComputerTools({
 			toolExecutors: this.runtimeSpec.toolExecutors,
-			coordinateSystem: this.runtimeSpec.coordinateSystem,
-			screenshot: this.runtimeSpec.screenshot,
-			computerUseExtra: this.options.computerUseExtra
-		}), ...this.options.extraTools ?? []];
+			computerUseExtra: this.options.computerUseExtra,
+			playwright: this.options.playwright
+		}, this.translator), ...this.options.extraTools ?? []];
 	}
 	onPayload() {
 		const runtimeSpec = this.runtimeSpec;
@@ -578,7 +607,11 @@ var CuaRuntimeController = class {
 		}) : void 0, this.options.onPayload);
 	}
 	keepToolNames() {
-		return [...(this.options.extraTools ?? []).map((tool) => tool.name), ...this.options.computerUseExtra ? [CUA_NAVIGATION_TOOL_NAME] : []];
+		return [
+			...(this.options.extraTools ?? []).map((tool) => tool.name),
+			...this.options.computerUseExtra ? [CUA_NAVIGATION_TOOL_NAME] : [],
+			...this.options.playwright ? [CUA_PLAYWRIGHT_TOOL_NAME] : []
+		];
 	}
 	createTranslator() {
 		return new InternalComputerTranslator({
@@ -605,15 +638,18 @@ async function getCuaEnvApiKeyAndHeaders(model) {
 var CuaAgent = class extends Agent {
 	runtime;
 	ownsSystemPrompt;
+	runtimeDirty = false;
 	stateProxy;
+	stateProxyTarget;
 	constructor(options) {
-		const { browser, client, initialState, onPayload, streamFn, prepareNextTurn, extraTools, computerUseExtra, ...agentOptions } = options;
+		const { browser, client, initialState, onPayload, streamFn, prepareNextTurn, extraTools, computerUseExtra, playwright, ...agentOptions } = options;
 		const runtime = new CuaRuntimeController({
 			browser,
 			client,
 			model: initialState.model,
 			extraTools,
 			computerUseExtra,
+			playwright,
 			onPayload
 		});
 		const wrappedStreamFn = (model, context, streamOptions) => {
@@ -638,13 +674,17 @@ var CuaAgent = class extends Agent {
 		this.runtime = runtime;
 		this.ownsSystemPrompt = initialState.systemPrompt === void 0;
 		/**
-		* pi calls `prepareNextTurn` between provider requests. Wrapping it lets CUA
-		* honor any user-provided turn update while also refreshing provider-specific
-		* defaults if that update changes the model.
+		* pi's loop only re-reads model/tools/prompt between provider requests
+		* through `prepareNextTurn`. The wrapper stays pass-through (returning
+		* `undefined`, i.e. stock pi behavior) until either the user hook returns
+		* an update or a mid-run model assignment marks the CUA runtime dirty —
+		* only then is a turn update built from current state.
 		*/
 		this.prepareNextTurn = async (signal) => {
 			const update = await prepareNextTurn?.(signal);
 			if (update?.model) this.applyRuntime(update.model);
+			if (!update && !this.runtimeDirty) return void 0;
+			this.runtimeDirty = false;
 			const state = super.state;
 			const context = update?.context ?? {
 				systemPrompt: state.systemPrompt,
@@ -668,17 +708,22 @@ var CuaAgent = class extends Agent {
 	* and payload hooks for the selected provider.
 	*/
 	get state() {
-		if (!this.stateProxy) this.stateProxy = new Proxy(super.state, { set: (target, prop, value, receiver) => {
-			if (prop === "model") {
-				this.applyRuntime(value);
-				return true;
-			}
-			return Reflect.set(target, prop, value, receiver);
-		} });
+		const target = super.state;
+		if (!this.stateProxy || this.stateProxyTarget !== target) {
+			this.stateProxyTarget = target;
+			this.stateProxy = new Proxy(target, { set: (proxied, prop, value, receiver) => {
+				if (prop === "model") {
+					this.applyRuntime(value);
+					return true;
+				}
+				return Reflect.set(proxied, prop, value, receiver);
+			} });
+		}
 		return this.stateProxy;
 	}
 	applyRuntime(model) {
 		this.runtime.setModel(model);
+		this.runtimeDirty = true;
 		const state = super.state;
 		state.model = this.runtime.model;
 		state.tools = this.runtime.tools();
@@ -697,13 +742,14 @@ var CuaAgentHarness = class extends AgentHarness {
 	runtime;
 	requestedActiveToolNames;
 	constructor(options) {
-		const { browser, client, model, extraTools, computerUseExtra, systemPrompt, getApiKeyAndHeaders, onPayload, activeToolNames, ...harnessOptions } = options;
+		const { browser, client, model, extraTools, computerUseExtra, playwright, systemPrompt, getApiKeyAndHeaders, onPayload, activeToolNames, ...harnessOptions } = options;
 		const runtime = new CuaRuntimeController({
 			browser,
 			client,
 			model,
 			extraTools,
 			computerUseExtra,
+			playwright,
 			onPayload
 		});
 		const resolvedTools = runtime.tools();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onkernel/cua-agent",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Kernel browser computer-use Agent and AgentHarness classes built on pi-agent-core",
   "license": "MIT",
   "type": "module",
@@ -42,7 +42,7 @@
   "dependencies": {
     "@earendil-works/pi-agent-core": "0.79.1",
     "@earendil-works/pi-ai": "0.79.1",
-    "@onkernel/cua-ai": "0.2.2",
+    "@onkernel/cua-ai": "0.3.1",
     "@onkernel/sdk": "0.49.0",
     "sharp": "^0.34.5"
   },