@alexkroman1/aai-ui 0.12.2 → 0.13.0

package/dist/session/types.d.ts

@@ -0,0 +1,476 @@
+/**
+ * Type declarations for the framework-agnostic voice session core.
+ *
+ * Split out of `session/browser-session.ts` to keep that module focused on behaviour.
+ */
+import type { Epoch } from "@alexkroman1/aai/internal";
+import type { SessionIdentity } from "../client-identity.ts";
+import type { AgentState, ChatMessage, SessionError, ToolCallInfo, WebSocketConstructor } from "../types.ts";
+import type { AudioPath } from "./audio-state.ts";
+/**
+ * A custom event emitted by the agent via `ctx.send(event, data)` — the
+ * payload the session records in `SessionSnapshot.customEvents` (`id` is a
+ * monotonic session-unique counter, `event` the name, `data` the payload).
+ *
+ * Deliberately NOT the DOM `CustomEvent`: it shares nothing with that
+ * interface, and the old name shadowed the global in `.tsx` files.
+ *
+ * @public
+ */
+export type AgentCustomEvent = {
+    readonly id: number;
+    readonly event: string;
+    readonly data: unknown;
+};
+/**
+ * The agent's projected state as the client holds it: one entry per slot the
+ * agent's `syncState` names, keyed by that slot's name.
+ *
+ * @public
+ */
+export type AgentStateFrame = Readonly<Record<string, unknown>>;
+/**
+ * Immutable snapshot of the session state.
+ *
+ * Consumers (e.g. React hooks via `useSyncExternalStore`) read this to render.
+ * A new object reference is created on every state change.
+ *
+ * @remarks
+ * **Four fields describe liveness and they answer different questions.** They
+ * are routinely collapsed into one truthy check, which is how a chrome ends up
+ * showing a live indicator over a call that has ended:
+ *
+ * | Field | The question it answers |
+ * | --- | --- |
+ * | `started` | Has the caller pressed Start? `end()` puts it back to `false`; `disconnect()` does not. |
+ * | `running` | Is the socket MEANT to be up? `toggle()` is what flips it. |
+ * | `recording` | Is the microphone actually live right now? |
+ * | `state` | What is the agent doing — see {@link AgentState}. |
+ *
+ * @public
+ */
+export type SessionSnapshot = {
+    /** What the agent is doing. See {@link AgentState} for the seven members. */
+    readonly state: AgentState;
+    /**
+     * True while the microphone is live and streaming to the server.
+     *
+     * This is the mic, not the session: a session can be `running` with the mic
+     * still opening, and a failure to acquire it leaves this `false` with the
+     * socket up.
+     */
+    readonly recording: boolean;
+    /**
+     * Whether the caller has MUTED the microphone — see
+     * {@link BrowserSession.setMicMuted}. Independent of `recording`: a muted
+     * mic stays open and keeps streaming (as silence), so `recording` stays
+     * `true`. UI state, so it survives reconnects, `disconnect()` and `end()`.
+     */
+    readonly micMuted: boolean;
+    /**
+     * The WebSocket URL a program can connect to directly — the long-living
+     * platform endpoint, e.g. `wss://host/my-agent/websocket`. Derived from
+     * `platformUrl` at construction — available before connecting — and never
+     * replaced by the brokered sandbox tunnel URL the session may actually be
+     * connected to: that URL is ephemeral (it dies when the sandbox is
+     * replaced), while the platform endpoint is stable and upgrades
+     * programmatic clients to the current sandbox endpoint itself.
+     */
+    readonly apiUrl: string;
+    /**
+     * Monotonically increasing counter bumped whenever rendered conversation
+     * content changes (`messages`, `toolCalls`, or either live transcript).
+     * Cheap dependency for scroll-to-bottom effects — unlike summed lengths it
+     * never collides when the capped arrays slide.
+     */
+    readonly contentVersion: number;
+    /**
+     * The conversation so far, oldest first — user and assistant turns only.
+     * Tool activity is NOT in here; it is in `toolCalls`. Capped, so the oldest
+     * entries slide off a long call.
+     */
+    readonly messages: ChatMessage[];
+    /**
+     * Every tool call the agent has made this session, in order, each carrying
+     * its own pending/settled state. Capped like `messages`. `useToolResult` and
+     * `useToolCallStart` are the narrow readers; this is the whole log.
+     */
+    readonly toolCalls: ToolCallInfo[];
+    /**
+     * Custom events the agent pushed with `ctx.send(event, data)`, in order.
+     * A LOG rather than a value — `useEvent(name, cb)` is the reader that
+     * delivers each one exactly once; reading this array directly means owning
+     * the cursor yourself.
+     */
+    readonly customEvents: AgentCustomEvent[];
+    /**
+     * Latest state the agent projected via `syncState` — keyed by slot name,
+     * `{ [slot]: view }` — or `null` before the first push. A value, not a log —
+     * a component that mounts mid-session reads current state rather than
+     * replaying events it missed. A slot whose view did not change keeps its
+     * previous value object, so a selector over one slot is stable across pushes.
+     */
+    readonly agentState: AgentStateFrame | null;
+    /**
+     * The caller's in-progress turn, as STT reports it.
+     *
+     * **`null` and `""` are different turns, and collapsing them is the mistake
+     * this field invites.** `null` is silence; `""` is speech DETECTED with no
+     * words back yet — where a live session sits for a few hundred milliseconds
+     * at the start of every turn. Read as one falsy check, the live-transcript
+     * row appears a beat late, on the first word rather than on the first sound,
+     * which is the moment it exists for.
+     *
+     * Prefer {@link useUserTranscript}, which returns the distinction as two
+     * named things (`speaking` to render on, `text` with a placeholder) rather
+     * than leaving each chrome to re-derive the ternary.
+     *
+     * Cleared when the turn is committed to `messages`.
+     */
+    readonly userTranscript: string | null;
+    /**
+     * The agent's reply as it streams, or `null` when it is not speaking.
+     * Cleared when the reply is committed to `messages`, so a chrome renders
+     * this row and the finished message, never both.
+     */
+    readonly agentTranscript: string | null;
+    /**
+     * The session's current failure, or `null`. Carries a `code`
+     * ({@link SessionErrorCode}), a message, and whether it was FATAL.
+     *
+     * A fatal error LATCHES: nothing clears it but the next completed handshake,
+     * because the frame announcing a session's death is also the frame that used
+     * to wipe the message explaining it. A non-fatal one is retired by later
+     * activity, which is what the recovery was written for.
+     */
+    readonly error: SessionError | null;
+    /**
+     * Whether the caller has pressed Start. `false` until the first `start()`,
+     * and back to `false` after `end()` — which is what makes a start-screen
+     * chrome show its Start control again. `disconnect()` leaves it `true`.
+     */
+    readonly started: boolean;
+    /**
+     * Whether the session is MEANT to be connected — the pause/resume state
+     * `toggle()` flips, not a report of the socket. A reconnecting session is
+     * still `running`.
+     */
+    readonly running: boolean;
+};
+/**
+ * The seal on a {@link BrowserSession}.
+ *
+ * TYPE-ONLY — there is no value at run time, so an object literal cannot carry
+ * the key and only `createBrowserSession` mints a session. That is what lets
+ * the handle grow a member without breaking anybody: push-to-talk added three
+ * REQUIRED methods and broke every hand-written double, which is the failure a
+ * handle a caller RECEIVES should not be able to cause.
+ *
+ * @public
+ */
+export declare const browserSessionBrand: unique symbol;
+/**
+ * Push-to-talk's three edges on a {@link BrowserSession} — `session.userTurn`.
+ *
+ * Only an agent declaring `turnTaking: { detection: "manual" }` honours them; any other
+ * agent logs once and ignores them, because its transcriber already ends each
+ * turn on a pause. `usePushToTalk` is the hook a button is built on, and the
+ * way a `client.tsx` reaches these: a sub-handle rather than three methods on
+ * the session, so a feature one agent in many declares is not three members
+ * every session carries in its autocomplete.
+ *
+ * @public
+ */
+export type UserTurnControls = {
+    /**
+     * OPEN a turn — the button went down. Stops the agent if it is speaking
+     * (discarding its queued audio here at once, rather than a round trip later)
+     * and lets the microphone through to the transcriber.
+     */
+    start(): void;
+    /** CLOSE the turn and have the agent answer everything said since `start()` — the button came up. */
+    commit(): void;
+    /**
+     * Close the turn and THROW AWAY what was said in it — a cancelled press (the
+     * pointer left the button, Escape). The agent answers nothing.
+     */
+    clear(): void;
+};
+/**
+ * How a page answers a tool call it ran — see {@link BrowserSession.sendToolResult}.
+ *
+ * @public
+ */
+export type ToolCallOutcome = {
+    result: unknown;
+} | {
+    error: string;
+};
+/**
+ * Options for {@link BrowserSession.sendText}.
+ *
+ * @public
+ */
+export type SendTextOptions = {
+    /**
+     * Open the session for this message if it is not up: `start()` before the
+     * first call, `toggle()` after a hang-up (so a resumable session RESUMES),
+     * then send once the server has configured it. Messages typed meanwhile are
+     * kept in order. They are DROPPED if the session stops before it is up — a
+     * hang-up, `end()`, a failed connection — so a message typed into a call
+     * that never came is not answered by a later one. Default `false`: sending
+     * while disconnected does nothing.
+     */
+    connect?: boolean | undefined;
+};
+/**
+ * A framework-agnostic voice session that manages WebSocket communication,
+ * audio capture/playback, and agent state transitions.
+ *
+ * Uses a subscribe/getSnapshot pattern (compatible with React's
+ * `useSyncExternalStore`). Implements `Disposable` for resource cleanup.
+ *
+ * @sealed Only `createBrowserSession` produces one — see
+ * {@link browserSessionBrand}. A test double for a component is a real
+ * session with no socket, not an object literal.
+ *
+ * @public
+ */
+export type BrowserSession = {
+    /** The seal — see {@link browserSessionBrand}. */
+    readonly [browserSessionBrand]: true;
+    /** Return the current immutable state snapshot. */
+    getSnapshot(): SessionSnapshot;
+    /** Subscribe to state changes. Returns an unsubscribe function. */
+    subscribe(callback: () => void): () => void;
+    /**
+     * Open a WebSocket connection to the server and begin audio capture,
+     * without touching the `started`/`running` flags — the low-level half of
+     * `start()`. Most UIs call `start()` (first activation) or `toggle()`
+     * (mute-style connect/disconnect) instead.
+     * @param options - Optional. `signal` is an AbortSignal that, when aborted, disconnects the session.
+     */
+    connect(options?: {
+        signal?: AbortSignal;
+    }): void;
+    /** Cancel the current agent turn and discard in-flight TTS audio. */
+    cancel(): void;
+    /** Push-to-talk's three edges — see {@link UserTurnControls}. */
+    readonly userTurn: UserTurnControls;
+    /**
+     * Who this session is to its agent — the base URL it dials, the client id it
+     * sends, this tab's inbox holder id, and the server's session id. What
+     * `useInbox()`, `useClientId()` and `useSessionId()` read; see
+     * {@link SessionIdentity}.
+     */
+    readonly identity: SessionIdentity;
+    /**
+     * Send a TYPED user turn: the agent answers `text` exactly as if the caller
+     * had said it — aloud, with tools, under any `turnTaking.detection`.
+     *
+     * Interrupts the agent if it is speaking or thinking (its queued audio is
+     * discarded here at once, as `cancel()` does). The message is NOT echoed into
+     * `messages` locally: the server reports it as the same
+     * `userTranscript.committed` a spoken turn produces, and that is what adds
+     * the row — so it is also what a resumed session replays.
+     *
+     * `text` is trimmed; an empty message sends nothing, and so does a call
+     * while disconnected — unless `options.connect` is set, which opens the
+     * session (see {@link SendTextOptions}) and sends once it is up. Text longer than `MAX_TRANSCRIPT_CHARS` (100,000) is refused by
+     * the server. A speech-to-speech agent cannot take a typed turn: the server
+     * logs a warning once and the message is ignored.
+     *
+     * @example
+     * ```ts
+     * declare const session: import("@alexkroman1/aai-ui").Session;
+     * session.sendText("What's the weather tomorrow?");
+     * ```
+     */
+    sendText(text: string, options?: SendTextOptions): void;
+    /**
+     * Answer a pending tool call the agent asked THIS PAGE to run — a
+     * `clientTool` on the server. `result` is JSON-serialized and becomes the
+     * value the model reads; `error` fails the call with that message. A no-op
+     * while disconnected, and ignored by the server for a call nothing waits on.
+     *
+     * Most pages use `useClientTool(name, handler)`, which calls this for them.
+     */
+    sendToolResult(toolCallId: string, outcome: ToolCallOutcome): void;
+    /**
+     * Mute or unmute the microphone WITHOUT dropping the session — the gate a
+     * hold-to-talk button over an agent with automatic turn detection is built
+     * on (mute by default, unmute while held).
+     *
+     * While muted the session keeps streaming frames of the same length and
+     * cadence, zero-filled, so the server's transcriber hears silence: its
+     * automatic endpointing closes the turn on release, and the stream's clock
+     * stays true. Purely local — no command is sent and a speaking agent is not
+     * interrupted (call `cancel()` for that).
+     *
+     * UI state rather than connection state: it survives reconnects, resumes,
+     * `disconnect()` → `connect()` and `end()`, and may be set before the first
+     * `connect()` so a session opens already muted. Read it back as
+     * `micMuted` on the snapshot.
+     *
+     * @example
+     * ```ts
+     * declare const session: import("@alexkroman1/aai-ui").Session;
+     * session.setMicMuted(true); // typed-only until the talk button is held
+     * ```
+     */
+    setMicMuted(muted: boolean): void;
+    /**
+     * Clear messages, transcripts, and error state while keeping the current
+     * connection (unlike `reset()`, which also reconnects).
+     */
+    resetState(): void;
+    /**
+     * Reset the session: clear state as `resetState()` does, then drop and
+     * reopen the connection for a fresh conversation.
+     */
+    reset(): void;
+    /** Close the WebSocket and release all audio resources. */
+    disconnect(): void;
+    /**
+     * Start the session for the first time: sets `started` and `running`, then
+     * connects. Use this for the initial "start conversation" action;
+     * afterwards `toggle()` is the pause/resume control.
+     */
+    start(): void;
+    /** Toggle between connected and disconnected states (after `start()`). */
+    toggle(): void;
+    /**
+     * End the call: close the connection, clear the conversation, and return
+     * to the not-started state (`started` flips back to false, so a
+     * start-screen UI shows its Start control again). Unlike `reset()` —
+     * which keeps the call live and only clears the conversation — the next
+     * `start()` mints a brand-new session: a new session id, fresh
+     * per-session tool state, greeting included.
+     */
+    end(): void;
+    /**
+     * End the current call and immediately begin a new one — `end()` then
+     * `start()`, which is what "New Conversation" means for an agent that keeps
+     * SESSION-SCOPED STATE.
+     *
+     * `reset()` is the one whose name suggests this and it is not the same
+     * thing: it clears the transcript and reconnects, but the reconnect carries
+     * the same `?sessionId=`, so every `sessionSlot` on the server survives —
+     * the game world, the incident board, the cart. A caller who asked to start
+     * over gets a blank transcript in front of the old state. This drops the
+     * session id, so the next connect mints a fresh one and the greeting plays
+     * again.
+     *
+     * Three templates had each written `session.end(); session.start();` with
+     * the same paragraph explaining why `reset()` was wrong; the six on the
+     * stock shell could not, because {@link Controls} called `reset()` for them.
+     *
+     * @example
+     * ```ts
+     * declare const session: import("@alexkroman1/aai-ui").Session;
+     * session.restart();
+     * ```
+     */
+    restart(): void;
+    /**
+     * Continue an EARLIER session — one of this client's past conversations, say,
+     * picked from a list — without reloading the page: hang up the current call
+     * if there is one, clear the transcript on screen, and connect presenting
+     * `sessionId` as `?sessionId=`, so the server resumes THAT session.
+     *
+     * The server restores it the way a reload does: the session's own turns come
+     * back in `history.restored` (a session bound to a `client` keeps its events
+     * after it ends), together with the client's other prior sessions as seed —
+     * never counting this one twice. The greeting is skipped, as on any resume.
+     * The id is stored like a `config` frame's, so a reload continues it too.
+     *
+     * Throws a `RangeError` for an id the server would not honor (1-128 letters,
+     * digits, `-` or `_`) BEFORE touching the current call. An id that names no
+     * session the server has starts a fresh conversation under it, greeting
+     * skipped — the same answer a stale bookmark gets.
+     *
+     * @example
+     * ```ts
+     * declare const session: import("@alexkroman1/aai-ui").Session;
+     * declare const picked: { sessionId: string };
+     * session.resume(picked.sessionId);
+     * ```
+     */
+    resume(sessionId: string): void;
+    /** Alias for `disconnect` for use with `using`. */
+    [Symbol.dispose](): void;
+};
+/**
+ * Shared mutable connection state.
+ *
+ * **It used to carry the audio path too**, and the four fields that went are
+ * worth naming because the shape recurs: a `voiceIO` slot, an
+ * `audioSetupInFlight` latch, a `generation` epoch whose only readers were the
+ * bring-up's four staleness checks, and the two pre-init buffers. Each was a
+ * question about a position the connection did not have, readable and writable
+ * from anywhere. They are the audio path's own statechart now — see
+ * `session/audio-state.ts`.
+ *
+ * What is left is what the CONNECTION really owns: the socket, the server's
+ * idle retirement, and the turn boundary.
+ */
+export type ConnState = {
+    ws: InstanceType<WebSocketConstructor> | null;
+    /**
+     * The server retired this session for idleness (`idle_timeout`), so the
+     * close that follows is EXPECTED and must not be retried.
+     *
+     * Without this the automatic reconnect would immediately re-open the
+     * session the server just reclaimed — a tab left open would cycle forever
+     * and the guest would never see zero sessions, which is the whole point of
+     * the timeout. Reconnecting is the user's call from here (the controls
+     * reconnect on demand).
+     */
+    retiredByServer: boolean;
+    /**
+     * Turn epoch, bumped at every turn boundary — a committed user turn, a
+     * barge-in, a reset, AND every audio-path teardown (`cleanupAudio`).
+     *
+     * Async playback completions (`settleWhenAudioDrained`) capture it and
+     * discard themselves when it has moved. The teardown bumps are why it lives
+     * here rather than as a closure local in the message handlers: a reply's
+     * drain resolves whenever the AudioContext closes, so a session torn down
+     * mid-speech (hang up, fatal error, reconnect) would otherwise have
+     * `state: "listening"` written over its "disconnected"/"error" a moment
+     * later — a dead session reporting a live mic.
+     */
+    turn: Epoch;
+};
+/**
+ * The two liveness fields at rest.
+ *
+ * `running` and `recording` ride with almost every state transition and were
+ * spread as a pair of literals at seven sites across three modules — the same
+ * shape `session/state.ts` folded `state` and `error` out of, one field
+ * short. Naming it relates them: a transition that ends the call says so once,
+ * and a reader looking for "what stops the mic" finds one symbol rather than a
+ * grep.
+ *
+ * It is deliberately NOT the whole snapshot patch — a transition still supplies
+ * its own `agentState.apply(...)` projection beside this.
+ */
+export declare const STOPPED: {
+    readonly running: false;
+    readonly recording: false;
+};
+/**
+ * A turn boundary: end the current turn and settle whatever it was playing.
+ *
+ * The two calls are one fact and were written out at four sites — `cancel()`
+ * and `reset()` here, `reply.cancelled` and `session.reset` on the server side
+ * — where the pair is load-bearing in both halves. The bump stops a stale drain
+ * continuation from stamping `"listening"` over a state the session has since
+ * moved to; the flush settles the interrupted turn's `done()` so it cannot
+ * strand.
+ *
+ * Two further sites bump WITHOUT flushing (`cleanupAudio`, a committed user
+ * transcript) and stay spelled out, which is the point of naming this one: a
+ * bump on its own now reads as a deliberate choice rather than a forgotten
+ * flush.
+ */
+export declare function bargeIn(conn: ConnState, audio: AudioPath): void;

package/dist/session/url.d.ts

@@ -0,0 +1,17 @@
+/**
+ * What the client reports about itself on every attempt: `?location=`,
+ * `?phone=` and `?client=`, each already resolved and trimmed (empty = not
+ * reported).
+ */
+export type ClientReport = {
+    location?: string | undefined;
+    phone?: string | undefined;
+    client?: string | undefined;
+};
+/** Build the session WebSocket URL from the platform URL and resume state. */
+export declare function buildWsUrl(platformUrl: string, sessionId?: string, report?: ClientReport): URL;
+/**
+ * Turn a broker-provided session URL (`sessionUrl` from `GET client-config`
+ * — the agent's live sandbox endpoint) into this attempt's connect URL.
+ */
+export declare function buildBrokeredWsUrl(sessionUrl: string, sessionId?: string, report?: ClientReport): URL;

package/dist/session/user-turn.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Push-to-talk's three edges on the browser session — `session.userTurn`'s
+ * `start`, `commit` and `clear` — for an agent that declares
+ * `turnTaking: { detection: "manual" }`; and `session.sendText`, a turn the caller TYPED.
+ *
+ * Split out of `session/browser-session.ts` at the source-length cap. Each is a frame to
+ * the server; what the server does with it is `aai-runtime`'s
+ * `aai-runtime/src/transports/pipeline/speech/manual-turn.ts`. The two things done HERE rather than waiting a
+ * round trip for are the reason these are more than `sendJson` calls: a press
+ * stops the agent's queued audio at once, and a discarded turn's caption is
+ * cleared because no transcript for it will ever arrive.
+ *
+ * @module
+ */
+import type { SessionCommand } from "@alexkroman1/aai/protocol";
+import type { SessionStateMachine } from "./state.ts";
+import type { SendTextOptions, SessionSnapshot, ToolCallOutcome, UserTurnControls } from "./types.ts";
+/** What the three edges need from the session around them. @internal */
+export type UserTurnDeps = {
+    snapshot: () => SessionSnapshot;
+    /** A socket is open and can carry a frame. */
+    connected: () => boolean;
+    /** The local turn boundary — see `bargeIn` in `session/types.ts`. */
+    bargeIn: () => void;
+    agentState: SessionStateMachine;
+    updateState: (partial: Partial<SessionSnapshot>) => void;
+    sendJson: (msg: SessionCommand) => void;
+    /**
+     * Typed turns waiting for a session — `sendText(text, { connect: true })`
+     * while none is up. OWNED by the session core, which empties it whenever the
+     * session stops being meant to run (`running: false`): a hang-up, `end()`, a
+     * terminal close. A message typed into a call that then failed must not be
+     * answered by the next one.
+     */
+    queued: string[];
+    /** Bring the session up if it is not meant to be running — `start()` or `toggle()`. */
+    open: () => void;
+};
+/**
+ * Everything a caller's turn can be started by: push-to-talk's three edges
+ * and a TYPED turn — `session.userTurn` and `session.sendText`. One builder
+ * over one set of deps, because both are the same kind of thing (a user turn
+ * the client, not the transcriber, delimits) and both interrupt the same way.
+ *
+ * @internal
+ */
+export declare function createUserInput(deps: UserTurnDeps): {
+    userTurn: UserTurnControls;
+    sendText: (text: string, options?: SendTextOptions) => void;
+    /** A session came up (a `config` frame): send what was typed while it was not. */
+    flushQueued: () => void;
+    sendToolResult: (toolCallId: string, outcome: ToolCallOutcome) => void;
+};
+/** Build the three push-to-talk edges. @internal */
+export declare function createUserTurnActions(deps: UserTurnDeps): UserTurnControls;