@onkernel/cua-agent 0.2.0 → 0.3.1

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 0.3.1 - 2026-06-11
+
+- Update the `@onkernel/cua-ai` dependency to 0.2.1.
+
+## 0.3.0 - 2026-06-10
+
+- Replaces the vendored pi-agent-core snapshot with the released `@earendil-works/pi-agent-core@0.79.1` dependency. The full pi surface is still re-exported, but it now tracks the published package instead of a frozen fork.
+- BREAKING: `harness.agent` is removed. It only existed in the vendored pre-release snapshot and never shipped in any pi-agent-core release; use `getModel()`, `getTools()`, and `getActiveTools()` instead.
+- BREAKING: `steer()`, `followUp()`, `nextTurn()`, and `setStreamOptions()` on the harness now return promises and must be awaited.
+- BREAKING: the harness `model_select` and `thinking_level_select` events are renamed `model_update` and `thinking_level_update`, and the `steeringMode`/`followUpMode` property accessors became `getSteeringMode()`/`setSteeringMode()`/`getFollowUpMode()`/`setFollowUpMode()` methods.
+- BREAKING: `ExecutionEnv` is now `Result`-based. Custom env implementations return `Result` values instead of throwing.
+- BREAKING: requires Node.js >= 22.19.0.
+- `NodeExecutionEnv` now comes from `@earendil-works/pi-agent-core`'s `/node` subpath; importing it from `@onkernel/cua-agent` keeps working.
+- Tool execution follows pi's throw-on-failure contract: failed browser actions throw an error labeled with the action instead of also encoding the failure into tool result content and details.
+- Moves the yutori screenshot payload append into `@onkernel/cua-ai`'s payload middleware.
+- Built ESM output uses explicit `.js` relative import specifiers so `dist` resolves under plain Node.js.
+
 ## 0.2.0 - 2026-05-13
 
 - Adds `CuaAgentHarness`, a provider-aware harness API with session-backed turns, resource and prompt helpers, active tool selection, and model switching.

package/README.md

@@ -1,7 +1,9 @@
 # `@onkernel/cua-agent`
 
-Kernel browser computer-use classes built on vendored pi `Agent` and
-`AgentHarness` source.
+Kernel browser computer-use classes built on the `Agent` and `AgentHarness`
+classes from [`@earendil-works/pi-agent-core`](https://www.npmjs.com/package/@earendil-works/pi-agent-core).
+The full pi-agent-core surface is re-exported from this package, including
+`NodeExecutionEnv` from its `/node` subpath.
 
 This package keeps pi agent semantics intact and adds browser execution
 plumbing for canonical CUA tools.
@@ -84,6 +86,8 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions:
 - `browser` (Kernel browser response)
 - `client` (Kernel SDK client)
 - CUA model refs (`"provider:model"`) accepted where pi expects a concrete model
+- `extraTools` to add your own pi tools alongside the built-in browser tools
+- `computerUseExtra: true` to let the model use a small navigation helper
 
 If auth callbacks are omitted, both classes default to CUA env var conventions:
 - OpenAI: `OPENAI_API_KEY`
@@ -94,9 +98,19 @@ If auth callbacks are omitted, both classes default to CUA env var conventions:
 
 ### Tool Defaults
 
-If tools are omitted, the classes install canonical CUA computer tool executors
-using runtime specs from `@onkernel/cua-ai`. If tools are provided, they are
-used exactly.
+By default, the classes install provider-selected CUA computer tool executors
+from `@onkernel/cua-ai`. Each provider decides which tool names the model sees;
+the matching executor adapter translates returned tool calls into canonical CUA
+actions that run against the Kernel browser.
+
+Use `extraTools` to add your own pi tools alongside the provider's
+computer-use tools. This is useful when the model needs to call
+application-specific code, such as looking up a record, writing a database row,
+or handing off to another service while it also controls the browser.
+
+`computerUseExtra: true` adds the `computer_use_extra` tool. Use it when you
+want one compact helper for common browser navigation/read operations:
+`goto`, `back`, `forward`, and `url`.
 
 ### Model Switching
 
@@ -111,8 +125,8 @@ the next provider request.
 
 ### Tool Composition
 
-Use `createCuaComputerTools()` to compose your own tool list from canonical
-tool definitions:
+Use `createCuaComputerTools()` to compose your own tool list from provider
+execution adapters:
 
 ```ts
 import { resolveCuaRuntimeSpec } from "@onkernel/cua-ai";
@@ -123,11 +137,12 @@ const tools = [
   ...createCuaComputerTools({
     browser,
     client,
-    toolDefinitions: runtime.toolDefinitions,
+    toolExecutors: runtime.toolExecutors,
   }),
   myCustomTool,
 ];
 ```
 
 For full event semantics, steering, follow-up queues, and tool execution
-details, see the pi agent core source vendored in this package.
+details, see the [`@earendil-works/pi-agent-core`](https://www.npmjs.com/package/@earendil-works/pi-agent-core)
+package.

package/dist/index.d.ts

@@ -1,8 +1,138 @@
-export * from "./vendor/pi-agent-core/index";
-export type { KernelBrowser } from "./translator/translator";
-export { createCuaComputerTools } from "./tools";
-export type { BatchDetails, ComputerToolOptions, CuaExecutorTool, NavigationDetails, SupportedCuaExecutorToolName, } from "./tools";
-export { SUPPORTED_CUA_EXECUTOR_TOOL_NAMES } from "./tools";
-export { CuaAgent, CuaAgentHarness } from "./agent";
-export type { CuaAgentHarnessOptions, CuaAgentOptions, CuaAgentState } from "./agent";
-//# sourceMappingURL=index.d.ts.map
+import { Agent, AgentHarness, AgentHarnessOptions, AgentOptions, AgentState, AgentTool, PromptTemplate, Skill } from "@earendil-works/pi-agent-core";
+import { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node";
+import { Api, ComputerToolCoordinateSystem, CuaModelRef, CuaScreenshotSpec, CuaToolExecutorSpec, Model, SimpleStreamOptions, TSchema } from "@onkernel/cua-ai";
+import Kernel from "@onkernel/sdk";
+import { BrowserCreateResponse, BrowserRetrieveResponse } from "@onkernel/sdk/resources/browsers";
+export * from "@earendil-works/pi-agent-core";
+
+//#region src/translator/translator.d.ts
+type KernelBrowser = BrowserCreateResponse | BrowserRetrieveResponse;
+//#endregion
+//#region src/tools.d.ts
+interface ComputerToolOptions {
+  browser: KernelBrowser;
+  client: Kernel;
+  toolExecutors: CuaToolExecutorSpec[];
+  coordinateSystem?: ComputerToolCoordinateSystem;
+  screenshot?: CuaScreenshotSpec;
+  computerUseExtra?: boolean;
+}
+interface BatchDetails {
+  statusText: string;
+  readResults: Array<{
+    type: "url";
+    url: string;
+  } | {
+    type: "screenshot";
+    bytes: number;
+  } | {
+    type: "cursor_position";
+    x: number;
+    y: number;
+  }>;
+}
+interface NavigationDetails {
+  action: string;
+  statusText: string;
+  url?: string;
+}
+type BatchTool = AgentTool<TSchema, BatchDetails>;
+type NavigationTool = AgentTool<TSchema, NavigationDetails>;
+type ActionTool = AgentTool<TSchema, BatchDetails>;
+type CuaExecutorTool = BatchTool | NavigationTool | ActionTool;
+declare function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTool[];
+//#endregion
+//#region src/agent.d.ts
+/** A CUA model reference string or a concrete pi model object. */
+type CuaRuntimeInput = CuaModelRef | Model<Api>;
+/**
+ * Agent state exposed by {@link CuaAgent}.
+ *
+ * It is the regular pi `AgentState`, except assigning `state.model` may use a
+ * CUA model ref such as `"openai:gpt-5.5"`. CUA-owned tools and the default
+ * system prompt are refreshed to match the new provider runtime.
+ */
+interface CuaAgentState extends Omit<AgentState, "model"> {
+  /** The concrete pi model currently used by the underlying agent loop. */
+  get model(): Model<Api>;
+  /** Assign a concrete pi model or CUA model ref and refresh CUA runtime defaults. */
+  set model(model: CuaRuntimeInput);
+}
+/** Initial state for {@link CuaAgent}. */
+type CuaAgentInitialState = Omit<NonNullable<AgentOptions["initialState"]>, "model" | "tools"> & {
+  /** Model to use for the first turn. CUA refs are resolved before pi sees the state. */model: CuaRuntimeInput;
+};
+/**
+ * Constructor options for {@link CuaAgent}.
+ *
+ * `browser` and `client` are used to build the default computer-use tools.
+ * Everything else follows pi `AgentOptions`, with `initialState.model`
+ * widened to accept CUA model refs.
+ */
+type CuaAgentOptions = Omit<AgentOptions, "initialState"> & {
+  /** Kernel browser session used by default CUA tools. */browser: KernelBrowser; /** Kernel SDK client used by default CUA tools. */
+  client: Kernel; /** Initial pi state plus a CUA-aware model value. */
+  initialState: CuaAgentInitialState; /** Add your own pi tools alongside the built-in browser tools. */
+  extraTools?: AgentTool[]; /** Expose a helper for browser navigation and URL reads. */
+  computerUseExtra?: boolean;
+};
+/**
+ * Constructor options for {@link CuaAgentHarness}.
+ *
+ * The harness keeps pi `AgentHarnessOptions` intact except that `model`
+ * accepts CUA refs and `browser`/`client` are required to build default
+ * computer-use tools. Callers provide pi's `env` and `session` directly.
+ */
+type CuaAgentHarnessOptions<TSkill extends Skill = Skill, TPromptTemplate extends PromptTemplate = PromptTemplate> = Omit<AgentHarnessOptions<TSkill, TPromptTemplate, AgentTool>, "model" | "tools"> & {
+  /** Kernel browser session used by default CUA tools. */browser: KernelBrowser; /** Kernel SDK client used by default CUA tools. */
+  client: Kernel; /** Model used by the harness. CUA refs are resolved before pi sees the model. */
+  model: CuaRuntimeInput; /** Add your own pi tools alongside the built-in browser tools. */
+  extraTools?: AgentTool[]; /** Expose a helper for browser navigation and URL reads. */
+  computerUseExtra?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */
+  onPayload?: SimpleStreamOptions["onPayload"];
+};
+/**
+ * Pi `Agent` configured for Kernel browser computer use.
+ *
+ * Use this class when you want direct access to the lower-level pi agent state,
+ * queues, event stream, and `state.model` mutation model. It resolves CUA model
+ * refs, installs provider-appropriate CUA tools by default, and keeps those
+ * defaults in sync when `agent.state.model` changes.
+ */
+declare class CuaAgent extends Agent {
+  private readonly runtime;
+  private readonly ownsSystemPrompt;
+  private stateProxy?;
+  constructor(options: CuaAgentOptions);
+  /**
+   * Return a state proxy so `agent.state.model = "provider:model"` can behave
+   * like pi's normal mutable state while also re-resolving CUA tools, prompt,
+   * and payload hooks for the selected provider.
+   */
+  get state(): CuaAgentState;
+  private applyRuntime;
+}
+/**
+ * Pi `AgentHarness` configured for Kernel browser computer use.
+ *
+ * Use this class when you want pi's higher-level harness APIs for sessions,
+ * resources, prompt templates, queue events, compaction, and model selection.
+ * It installs provider CUA tools by default and keeps CUA-owned runtime
+ * defaults in sync through `setModel()`.
+ */
+declare class CuaAgentHarness<TSkill extends Skill = Skill, TPromptTemplate extends PromptTemplate = PromptTemplate> extends AgentHarness<TSkill, TPromptTemplate, AgentTool> {
+  private readonly runtime;
+  private requestedActiveToolNames?;
+  constructor(options: CuaAgentHarnessOptions<TSkill, TPromptTemplate>);
+  /**
+   * Mirror pi `AgentHarness.setModel()` while accepting CUA model refs.
+   *
+   * The override refreshes CUA-owned tools before delegating to pi so the
+   * harness snapshot and session model-change entry are written with the
+   * concrete model selected by `@onkernel/cua-ai`.
+   */
+  setModel(model: CuaRuntimeInput): Promise<void>;
+  setActiveTools(toolNames: string[]): Promise<void>;
+}
+//#endregion
+export { type BatchDetails, type ComputerToolOptions, CuaAgent, CuaAgentHarness, type CuaAgentHarnessOptions, type CuaAgentOptions, type CuaAgentState, type CuaExecutorTool, type KernelBrowser, type NavigationDetails, NodeExecutionEnv, createCuaComputerTools };