@namzu/sdk 45.0.0 → 46.0.0

package/src/types/browser/index.ts

@@ -0,0 +1,341 @@
+// ---------------------------------------------------------------------------
+// The browser contract: what a browser host does for the `browser` and
+// `browser_act` tools. The SDK owns the model-facing tools and this
+// interface; a host package (for example `@namzu/browser`) owns the engine.
+// ---------------------------------------------------------------------------
+
+/** Every action the `browser` tool (observe and navigate) can ask for. */
+export type BrowserObserveActionName =
+	| 'navigate'
+	| 'back'
+	| 'forward'
+	| 'reload'
+	| 'snapshot'
+	| 'screenshot'
+	| 'scroll'
+	| 'wait_for'
+	| 'tabs'
+
+/** Every action the `browser_act` tool (change the page) can ask for. */
+export type BrowserActActionName =
+	| 'click'
+	| 'type'
+	| 'fill_form'
+	| 'select'
+	| 'press'
+	| 'hover'
+	| 'upload'
+	| 'dialog'
+
+export type BrowserActionName = BrowserObserveActionName | BrowserActActionName
+
+export type BrowserScrollDirection = 'up' | 'down' | 'left' | 'right'
+
+export type BrowserTabsOp = 'list' | 'select' | 'close' | 'new'
+
+// ---------------------------------------------------------------------------
+// Capabilities — frozen at host construction; the model reads them through
+// the tools' descriptions and schemas.
+// ---------------------------------------------------------------------------
+
+export interface BrowserCapabilities {
+	/** Which engine drives the browser, in words: `local-chromium`, `windows-cdp`. */
+	readonly engine: string
+	/** The browser window is not shown. */
+	readonly headless: boolean
+	/** `screenshot` can return an image. */
+	readonly screenshot: boolean
+	/** `upload` can attach a local file to a file input. */
+	readonly upload: boolean
+	/**
+	 * Exact action subset when known. Absent: every action whose broad flag
+	 * above allows it. Actions outside the subset are removed from the model
+	 * schema and refused before the host is called.
+	 */
+	readonly supportedActions?: readonly BrowserActionName[]
+	/**
+	 * The most snapshot text one call returns. The tool cuts anything longer.
+	 * Absent: {@link BROWSER_SNAPSHOT_MAX_CHARS}.
+	 */
+	readonly snapshotMaxChars?: number
+	/**
+	 * Why the browser cannot be used at all — the engine is not installed,
+	 * the profile is missing. Present, both tools stay mounted, say this in
+	 * their descriptions and refuse every call with it, so the model reads
+	 * the reason once and tells the user instead of retrying.
+	 */
+	readonly unavailableReason?: string
+}
+
+/** Default and ceiling of snapshot text per call, in characters. */
+export const BROWSER_SNAPSHOT_MAX_CHARS = 20_000
+
+/** Longest `wait_for` the tool accepts, in milliseconds. */
+export const BROWSER_WAIT_MAX_MS = 30_000
+
+/** Most fields one `fill_form` call may set. */
+export const BROWSER_FILL_FORM_MAX_FIELDS = 20
+
+// ---------------------------------------------------------------------------
+// Actions
+// ---------------------------------------------------------------------------
+
+/**
+ * What the `browser` tool asks of the host. Every URL here has already been
+ * canonicalised by the tool (see `canonicalizeBrowserUrl`): only http, https
+ * or `about:blank`, no credentials, never a cloud metadata address.
+ */
+export type BrowserObserveAction =
+	| { readonly action: 'navigate'; readonly url: string }
+	| { readonly action: 'back' }
+	| { readonly action: 'forward' }
+	| { readonly action: 'reload' }
+	| { readonly action: 'snapshot'; readonly ref?: string; readonly cursor?: string }
+	| { readonly action: 'screenshot'; readonly ref?: string; readonly fullPage?: boolean }
+	| {
+			readonly action: 'scroll'
+			readonly direction: BrowserScrollDirection
+			readonly ref?: string
+			/** Screens to scroll; the host's default when absent. */
+			readonly amount?: number
+	  }
+	| {
+			readonly action: 'wait_for'
+			readonly text?: string
+			readonly textGone?: string
+			readonly timeMs?: number
+	  }
+	| {
+			readonly action: 'tabs'
+			readonly op: BrowserTabsOp
+			/** `select` and `close`: the tab id from `list` or a page header. */
+			readonly tab?: string
+			/** `new`: the address to open; `about:blank` when absent. */
+			readonly url?: string
+	  }
+
+export interface BrowserFormField {
+	readonly ref: string
+	/** Text for a text box; `true`/`false` for a checkbox; the option's label for a select. */
+	readonly value: string
+}
+
+/** The change a `browser_act` call makes. */
+export type BrowserActOperation =
+	| { readonly action: 'click'; readonly ref: string; readonly doubleClick?: boolean }
+	| {
+			readonly action: 'type'
+			readonly ref: string
+			readonly text: string
+			/** Press Enter after typing. */
+			readonly submit?: boolean
+	  }
+	| { readonly action: 'fill_form'; readonly fields: readonly BrowserFormField[] }
+	| { readonly action: 'select'; readonly ref: string; readonly values: readonly string[] }
+	| { readonly action: 'press'; readonly key: string; readonly ref?: string }
+	| { readonly action: 'hover'; readonly ref: string }
+	| { readonly action: 'upload'; readonly ref: string; readonly path: string }
+	| { readonly action: 'dialog'; readonly accept: boolean; readonly promptText?: string }
+
+/**
+ * What the `browser_act` tool asks of the host.
+ *
+ * `origin` is the page origin the model read from the snapshot header
+ * (`Page: <origin> — …`), canonicalised by the tool. **The host MUST compare
+ * it with the live origin of the page it is about to act on, immediately
+ * before acting, and throw a {@link BrowserOriginMismatch} without acting
+ * when they differ.** That comparison is what binds an approval of "click
+ * Place order on shop.example.com" to shop.example.com: a redirect between
+ * the snapshot and the click must not carry the click to another site.
+ */
+export type BrowserActAction = BrowserActOperation & {
+	readonly origin: string
+	/** Return a snapshot of the page after the action. */
+	readonly snapshot?: boolean
+}
+
+// ---------------------------------------------------------------------------
+// Results
+// ---------------------------------------------------------------------------
+
+/** The page an action left the browser on, as the host observed it. */
+export interface BrowserPageInfo {
+	/** Canonical origin (`https://github.com`), or `null` for `about:blank`. */
+	readonly origin: string
+	readonly url: string
+	/** The document title. Page-controlled: the tool quotes and cuts it. */
+	readonly title: string
+	/** The tab's id, e.g. `t1`. */
+	readonly tab: string
+}
+
+export interface BrowserTabInfo extends BrowserPageInfo {
+	readonly active: boolean
+}
+
+export interface BrowserSnapshot {
+	readonly page: BrowserPageInfo
+	/**
+	 * The accessibility tree as text, one element per line, each actionable
+	 * element carrying `[ref=eN]`. Page-controlled: the tool wraps it as
+	 * untrusted content.
+	 */
+	readonly text: string
+	/** Present when more text follows; pass it back as `cursor`. */
+	readonly nextCursor?: string
+}
+
+export interface BrowserScreenshot {
+	readonly page: BrowserPageInfo
+	readonly data: Uint8Array
+	readonly mimeType: 'image/png' | 'image/jpeg'
+	readonly width: number
+	readonly height: number
+}
+
+/**
+ * What an action produced. Every field is optional because actions differ;
+ * the tool renders whichever are present.
+ */
+export interface BrowserResult {
+	/** The page after the action. */
+	readonly page?: BrowserPageInfo
+	/**
+	 * The host's own note, in the host's words — "download of report.pdf
+	 * cancelled", "dialog accepted". Never page text: the tool shows it
+	 * outside the untrusted envelope.
+	 */
+	readonly message?: string
+	readonly snapshot?: BrowserSnapshot
+	readonly screenshot?: BrowserScreenshot
+	readonly tabs?: readonly BrowserTabInfo[]
+}
+
+/** What a snapshot said about one ref, for labelling a call before it runs. */
+export interface BrowserRefDescription {
+	/** ARIA role: `button`, `link`, `textbox`. */
+	readonly role: string
+	/** Accessible name. Page-controlled. */
+	readonly name?: string
+}
+
+/** Who and where, for labels and reviews. May change between calls. */
+export interface BrowserSessionInfo {
+	/** The profile the browser runs under. */
+	readonly profile?: string
+	/** Canonical origin of the active tab, when the host knows it. */
+	readonly origin?: string
+}
+
+// ---------------------------------------------------------------------------
+// Structural errors. Hosts throw values carrying these shapes; the tools
+// recognise them by shape, so a separately installed host and SDK need not
+// share an error class.
+// ---------------------------------------------------------------------------
+
+/** The live page is not on the origin a `browser_act` call named. Nothing was done. */
+export interface BrowserOriginMismatch {
+	readonly code: 'browser_origin_mismatch'
+	readonly expected: string
+	readonly actual: string
+	readonly message: string
+}
+
+/** The ref is not on the current page: it came from an older snapshot. Nothing was done. */
+export interface BrowserStaleRef {
+	readonly code: 'browser_stale_ref'
+	readonly ref: string
+	readonly message: string
+}
+
+/** Why the page needs a person. */
+export type BrowserHumanRequiredReason =
+	| 'sign-in'
+	| 'two-factor'
+	| 'captcha'
+	| 'bot-block'
+	| 'http-auth'
+	| 'credential-field'
+
+/**
+ * The page needs a person: a sign-in, a second factor, a CAPTCHA, a bot
+ * wall, or a field that takes a password or one-time code. The agent must
+ * stop and hand over; it never signs in, solves or types a credential.
+ */
+export interface BrowserHumanRequired {
+	readonly code: 'browser_human_required'
+	readonly reason: BrowserHumanRequiredReason
+	readonly origin: string
+	readonly message: string
+	readonly profile?: string
+	/** The command that opens a visible window on this profile, e.g. `namzu browser login work https://…`. */
+	readonly loginCommand?: string
+}
+
+/**
+ * A page-changing action started and did not report a clean completion. The
+ * page may already have changed, so replaying it is unsafe.
+ */
+export interface BrowserOutcomeUnknown {
+	readonly code: 'browser_outcome_unknown'
+	readonly action: BrowserActionName
+	readonly outcome: 'unknown'
+	readonly retrySafety: 'unsafe'
+	readonly message: string
+}
+
+/** The site policy does not allow this origin at the level the call needs. */
+export interface BrowserSiteDenied {
+	readonly code: 'browser_site_denied'
+	readonly origin: string
+	readonly message: string
+}
+
+export type BrowserHostError =
+	| BrowserOriginMismatch
+	| BrowserStaleRef
+	| BrowserHumanRequired
+	| BrowserOutcomeUnknown
+	| BrowserSiteDenied
+
+// ---------------------------------------------------------------------------
+// Host
+// ---------------------------------------------------------------------------
+
+export interface BrowserCallOptions {
+	/** Fires when the call is cancelled or times out. */
+	readonly signal?: AbortSignal
+}
+
+/**
+ * A browser the tools drive. Implementations live outside `@namzu/sdk`.
+ *
+ * The host is where the site policy is enforced after the fact: whatever the
+ * gate approved, the host re-checks the page a navigation, redirect or popup
+ * actually landed on, and the live origin before every `act`.
+ */
+export interface BrowserHost {
+	readonly id: string
+	readonly capabilities: BrowserCapabilities
+
+	/** Observe or navigate. Throws a {@link BrowserHostError} shape to refuse. */
+	observe(action: BrowserObserveAction, options?: BrowserCallOptions): Promise<BrowserResult>
+	/**
+	 * Change the page. MUST check `action.origin` against the live page first
+	 * (see {@link BrowserActAction}). Throws a {@link BrowserHostError} shape
+	 * to refuse.
+	 */
+	act(action: BrowserActAction, options?: BrowserCallOptions): Promise<BrowserResult>
+
+	/**
+	 * What the most recent snapshot said about `ref`, synchronously and
+	 * without touching the page, for the label a person approves. Undefined
+	 * when the ref is unknown.
+	 */
+	describeRef?(ref: string): BrowserRefDescription | undefined
+	/** The current profile and page, synchronously, for labels. */
+	session?(): BrowserSessionInfo
+
+	initialize?(): Promise<void>
+	dispose?(): Promise<void>
+}

package/src/types/computer-use/index.ts

@@ -44,10 +44,41 @@ export interface ComputerUseCapabilities {
 	 * model reads the reason once and does not try again.
 	 */
 	readonly unavailableReason?: string
+	/**
+	 * The host implements {@link ComputerUseHost.listWindows} and
+	 * {@link ComputerUseHost.focusWindow}. Absent or false, the tool does not
+	 * offer `list_windows` or `focus_window`, even when the methods exist.
+	 */
+	readonly windows?: boolean
+	/**
+	 * The host implements {@link ComputerUseHost.captureRegion}. Absent or
+	 * false, `zoom` still works: the tool crops a full capture instead, which
+	 * costs one whole-display capture per zoom.
+	 */
+	readonly regionCapture?: boolean
+	/**
+	 * The host implements {@link ComputerUseHost.uiSnapshot} and
+	 * {@link ComputerUseHost.uiAct}: an accessibility tree (Windows UI
+	 * Automation, macOS AX, AT-SPI, or a driver that wraps one) whose
+	 * elements can be acted on by reference instead of by pixel.
+	 *
+	 * @experimental The UI-tree surface is reserved for the host packages that
+	 * implement it; its shape may still change in a minor release.
+	 */
+	readonly uiTree?: boolean
 }
 
 // ---------------------------------------------------------------------------
 // Geometry + screenshot payload
+//
+// Units, once for the whole file: every coordinate and size a host accepts or
+// returns is in PHYSICAL pixels — the pixels of the captured bitmap, not
+// logical points or DPI-scaled units. A 3440x1440 monitor at 150 % scaling is
+// 3440x1440 here. A point a host is asked to act on is relative to the
+// top-left of the display it last captured (the primary display until a host
+// offers display selection); the host adds that display's origin itself.
+// Window and UI-element bounds are the exception and say so: they are in
+// virtual-desktop physical pixels, because a window can span displays.
 // ---------------------------------------------------------------------------
 
 export interface DisplayGeometry {
@@ -56,11 +87,144 @@ export interface DisplayGeometry {
 	readonly scaleFactor: number
 }
 
+/**
+ * The display a capture shows.
+ *
+ * `x`/`y` place it in the virtual desktop (a monitor left of the primary has
+ * a negative `x`); `width`/`height` are its physical size. `scaleFactor` is
+ * physical pixels per logical pixel — 1 at 96 DPI on Windows, 1.5 at 150 %,
+ * 2 on a Retina panel — reported for the host UI and for diagnosis; the tool
+ * never multiplies by it, because every coordinate crossing this interface
+ * is already physical.
+ */
+export interface DisplayInfo {
+	/** Stable for the host's lifetime; what a host would accept to select this display. */
+	readonly id: string
+	readonly x: number
+	readonly y: number
+	readonly width: number
+	readonly height: number
+	readonly scaleFactor: number
+	/** True for the display the operating system calls primary. */
+	readonly primary?: boolean
+}
+
 export interface ScreenshotResult {
 	readonly data: Buffer
 	readonly mimeType: 'image/png'
+	/** Physical pixel width of `data`. */
 	readonly width: number
+	/** Physical pixel height of `data`. */
 	readonly height: number
+	/**
+	 * The display this capture shows. A host should always set it; one written
+	 * before it existed does not, and the tool then assumes a single display at
+	 * the origin whose size is the capture's own, with a scale factor of 1.
+	 */
+	readonly display?: DisplayInfo
+}
+
+/** A rectangle in physical pixels. What its origin is relative to depends on where it appears. */
+export interface Rect {
+	readonly x: number
+	readonly y: number
+	readonly width: number
+	readonly height: number
+}
+
+/** One top-level window, as {@link ComputerUseHost.listWindows} reports it. */
+export interface WindowInfo {
+	/** Opaque and host-defined (an HWND in hex, a CGWindowID, an X11 window id); valid for {@link ComputerUseHost.focusWindow}. */
+	readonly id: string
+	readonly title: string
+	/** Application or process name, e.g. `msedge`, `Teams`, `Code`. */
+	readonly app: string
+	readonly pid: number
+	/** Virtual-desktop physical pixels — not display-relative. */
+	readonly bounds: Rect
+	readonly focused: boolean
+	readonly minimized: boolean
+}
+
+/**
+ * What {@link ComputerUseHost.focusWindow} achieved. Bringing a window to
+ * the front is a request the operating system can refuse (Windows'
+ * foreground lock is the common case), so a host reports the window that is
+ * actually in front afterwards rather than assuming its request held.
+ */
+export interface FocusWindowResult {
+	/** True only when `focusedId` is the requested window. */
+	readonly ok: boolean
+	/** The window in front after the attempt, or null when none could be read. */
+	readonly focusedId: string | null
+}
+
+/**
+ * An action an accessibility element can take by reference.
+ *
+ * @experimental See {@link ComputerUseCapabilities.uiTree}.
+ */
+export type UiElementAction =
+	| 'invoke'
+	| 'focus'
+	| 'set_value'
+	| 'toggle'
+	| 'select'
+	| 'expand'
+	| 'collapse'
+	| 'scroll_into_view'
+
+/**
+ * One accessibility element.
+ *
+ * @experimental See {@link ComputerUseCapabilities.uiTree}.
+ */
+export interface UiElement {
+	/**
+	 * Opaque reference for {@link ComputerUseHost.uiAct}, valid until the
+	 * host's next snapshot. Empty for an element the host cannot act on (a
+	 * label, a group), which is in the tree for what it says.
+	 */
+	readonly ref: string
+	/** Platform role, e.g. `Button`, `Edit`, `ListItem` (UIA ControlType) or `AXButton`. */
+	readonly role: string
+	readonly name: string
+	readonly value?: string
+	/** A platform automation id, when the element has one. */
+	readonly automationId?: string
+	/** Virtual-desktop physical pixels, when the element is on screen. */
+	readonly bounds?: Rect
+	/** Element states such as `focused`, `disabled`, `selected`, `checked`, `expanded`. */
+	readonly states?: readonly string[]
+	readonly actions?: readonly UiElementAction[]
+	readonly children?: readonly UiElement[]
+}
+
+/**
+ * An accessibility tree for one window.
+ *
+ * @experimental See {@link ComputerUseCapabilities.uiTree}.
+ */
+export interface UiSnapshot {
+	readonly windowId?: string
+	/** The window's title, as its application sets it. */
+	readonly title?: string
+	/** The owning application or process name, as in {@link WindowInfo.app}. */
+	readonly app?: string
+	readonly root: UiElement
+	/** True when the host stopped walking the tree before it ended (a size or time bound). */
+	readonly truncated?: boolean
+}
+
+/**
+ * What {@link ComputerUseHost.uiAct} did.
+ *
+ * @experimental See {@link ComputerUseCapabilities.uiTree}.
+ */
+export interface UiActResult {
+	readonly ok: boolean
+	/** Why it did not, in words the model can act on (a stale ref, a disabled element). */
+	readonly detail?: string
 }
 
 export interface Point {
@@ -152,8 +316,46 @@ export interface ComputerUseHost {
 	readonly capabilities: ComputerUseCapabilities
 
 	getDisplayGeometry(): Promise<DisplayGeometry>
+	/**
+	 * Points in `action` are physical pixels relative to the display of the
+	 * most recent capture; a `screenshot` result carries that display in
+	 * {@link ScreenshotResult.display}.
+	 */
 	execute(action: ComputerUseAction): Promise<ComputerUseResult>
 
+	/**
+	 * The visible, titled top-level windows, front to back where the platform
+	 * knows the order. Offered to the model only with
+	 * {@link ComputerUseCapabilities.windows}.
+	 */
+	listWindows?(): Promise<readonly WindowInfo[]>
+	/**
+	 * Bring a window to the front, restoring it when minimized. Offered only
+	 * with {@link ComputerUseCapabilities.windows}.
+	 */
+	focusWindow?(id: string): Promise<FocusWindowResult>
+	/**
+	 * Capture one region of the current display at full physical resolution.
+	 * `rect` is display-relative physical pixels; the result's `width` and
+	 * `height` are the region's. Used by `zoom` when
+	 * {@link ComputerUseCapabilities.regionCapture} is set.
+	 */
+	captureRegion?(rect: Rect): Promise<ScreenshotResult>
+	/**
+	 * The accessibility tree of one window, or of the focused window when
+	 * `windowId` is omitted. Offered only with {@link ComputerUseCapabilities.uiTree}.
+	 *
+	 * @experimental
+	 */
+	uiSnapshot?(windowId?: string): Promise<UiSnapshot>
+	/**
+	 * Act on an element from the latest {@link uiSnapshot}. `value` is the text
+	 * for `set_value`. Offered only with {@link ComputerUseCapabilities.uiTree}.
+	 *
+	 * @experimental
+	 */
+	uiAct?(ref: string, action: UiElementAction, value?: string): Promise<UiActResult>
+
 	initialize?(): Promise<void>
 	dispose?(): Promise<void>
 }

package/src/types/session/events.ts

@@ -534,6 +534,12 @@ type CoreSessionEvent =
 			providerError?: import('../provider/error.js').ProviderErrorInfo
 			/** Curated operator copy, absent when no catalog rule matched. */
 			explanation?: { id: string; message: string; hint: string }
+			/**
+			 * Present when a tool asked for a person: its `ToolResult.handoff`.
+			 * The results of the batch are already committed; resuming the turn
+			 * calls the model with them.
+			 */
+			handoff?: import('../tool/index.js').ToolHandoff
 	  }
 	/** A paused turn continues from its checkpoint, under the same `turnId`. */
 	| {

package/src/types/session/records.ts

@@ -201,6 +201,15 @@ const providerError = z
 
 const explanation = z.object({ id: text, message: text, hint: text }).strict()
 
+/** A tool's request for a person, as `turn_paused` carries it. */
+const toolHandoff = z
+	.object({
+		kind: z.literal('human-required'),
+		reason: text,
+		detail: z.record(text).optional(),
+	})
+	.strict()
+
 const stopReason = z.enum([
 	'end_turn',
 	'token_budget',
@@ -361,6 +370,7 @@ export const TurnPausedRecordSchema = inTurn('turn_paused', {
 	providerError: providerError.optional(),
 	explanation: explanation.optional(),
 	budget: tokenBudgetSummary.optional(),
+	handoff: toolHandoff.optional(),
 })
 
 export const TurnResumingRecordSchema = inTurn('turn_resuming', {