@alexkroman1/aai-ui 1.8.3 → 1.9.0

package/dist/session-core-types.d.ts

@@ -0,0 +1,147 @@
+/**
+ * Type declarations for the framework-agnostic voice session core.
+ *
+ * Split out of `session-core.ts` to keep that module focused on behaviour.
+ * The public types here are re-exported from `session-core.ts` for
+ * backwards compatibility.
+ */
+import type { VoiceIO } from "./audio.ts";
+import type { AgentState, ChatMessage, SessionError, ToolCallInfo, VoiceSessionOptions, WebSocketConstructor } from "./types.ts";
+/**
+ * A custom event emitted by the agent via `ctx.send`.
+ *
+ * @public
+ */
+export type CustomEvent = {
+    readonly id: number;
+    readonly event: string;
+    readonly data: unknown;
+};
+/**
+ * Immutable snapshot of the session state.
+ *
+ * Consumers (e.g. React hooks via `useSyncExternalStore`) read this to render.
+ * A new object reference is created on every state change.
+ *
+ * @public
+ */
+export type SessionSnapshot = {
+    readonly state: AgentState;
+    /**
+     * False when the server declared the session text-only (`tts: none()`):
+     * no audio frames will arrive and replies render as text. True until the
+     * server's `config` message says otherwise (voice is the default).
+     */
+    readonly audioOut: boolean;
+    /**
+     * True while the microphone is live and streaming to the server. Voice
+     * sessions record for their whole lifetime; text-only sessions toggle this
+     * via `startRecording()` / `stopRecording()` (the record button).
+     */
+    readonly recording: boolean;
+    /**
+     * The WebSocket URL a program can connect to directly (the same endpoint
+     * this session uses), e.g. `wss://host/my-agent/websocket`. Derived from
+     * `platformUrl` at construction — available before connecting.
+     */
+    readonly apiUrl: string;
+    /**
+     * Monotonically increasing counter bumped whenever rendered conversation
+     * content changes (`messages`, `toolCalls`, or either live transcript).
+     * Cheap dependency for scroll-to-bottom effects — unlike summed lengths it
+     * never collides when the capped arrays slide.
+     */
+    readonly contentVersion: number;
+    readonly messages: ChatMessage[];
+    readonly toolCalls: ToolCallInfo[];
+    readonly customEvents: CustomEvent[];
+    readonly userTranscript: string | null;
+    readonly agentTranscript: string | null;
+    readonly error: SessionError | null;
+    readonly started: boolean;
+    readonly running: boolean;
+};
+/**
+ * A framework-agnostic voice session that manages WebSocket communication,
+ * audio capture/playback, and agent state transitions.
+ *
+ * Uses a subscribe/getSnapshot pattern (compatible with React's
+ * `useSyncExternalStore`). Implements `Disposable` for resource cleanup.
+ *
+ * @public
+ */
+export type SessionCore = {
+    /** Return the current immutable state snapshot. */
+    getSnapshot(): SessionSnapshot;
+    /** Subscribe to state changes. Returns an unsubscribe function. */
+    subscribe(callback: () => void): () => void;
+    /**
+     * Open a WebSocket connection to the server and begin audio capture.
+     * @param options - Optional. `signal` is an AbortSignal that, when aborted, disconnects the session.
+     */
+    connect(options?: {
+        signal?: AbortSignal;
+    }): void;
+    /** Cancel the current agent turn and discard in-flight TTS audio. */
+    cancel(): void;
+    /** Clear messages, transcript, and error state without disconnecting. */
+    resetState(): void;
+    /** Reset the session: clear state and reconnect. */
+    reset(): void;
+    /** Close the WebSocket and release all audio resources. */
+    disconnect(): void;
+    /** Start the session for the first time (sets `started` and `running`). */
+    start(): void;
+    /** Toggle between connected and disconnected states. */
+    toggle(): void;
+    /**
+     * Start streaming microphone audio (text-only sessions). Requests mic
+     * access on first use. No-op in voice sessions, where the mic is always on.
+     */
+    startRecording(): void;
+    /** Stop streaming microphone audio (text-only sessions). No-op in voice sessions. */
+    stopRecording(): void;
+    /**
+     * Decode an audio file (any format the browser can decode), resample it to
+     * the session's STT rate, and stream it to the server for transcription.
+     * Text-only sessions (`tts: none()`) only — voice sessions stream the
+     * microphone instead and reject. Also rejects when no session is connected,
+     * the mic is recording, another upload is in flight, or the file cannot be
+     * decoded. Resolves once the audio has been handed to the socket.
+     */
+    sendAudioFile(file: Blob): Promise<void>;
+    /** Alias for `disconnect` for use with `using`. */
+    [Symbol.dispose](): void;
+};
+export type SessionCoreOptions = VoiceSessionOptions;
+/**
+ * Shared mutable connection state for audio initialization.
+ *
+ * Tracks the active WebSocket, VoiceIO instance, and a generation counter
+ * that prevents stale async operations (e.g. a slow `initAudioCapture`) from
+ * assigning their results to a newer connection after a reconnect.
+ */
+export type ConnState = {
+    ws: InstanceType<WebSocketConstructor> | null;
+    voiceIO: VoiceIO | null;
+    audioSetupInFlight: boolean;
+    /** Monotonically increasing counter bumped on each connect(). Prevents a stale
+     *  initAudioCapture from assigning its voiceIO to a newer connection. */
+    generation: number;
+    /** Audio chunks that arrived before `voiceIO` was initialized — drained into
+     *  the playback worklet once init completes. Closes the race between the
+     *  server starting greeting audio (immediately on S2S connect) and the
+     *  client awaiting mic permission + worklet registration. */
+    preInitAudio: Uint8Array[];
+    /** True if `audio_done` arrived before `voiceIO` was initialized. The done
+     *  signal must be replayed after draining preInitAudio, or a short greeting
+     *  buffered during mic-permission never finishes playing. */
+    preInitDone: boolean;
+    /** The server's `config` payload for the current connection — kept so
+     *  text-only sessions can init the mic lazily (record button) and file
+     *  uploads know the STT sample rate to resample to. */
+    readyConfig: {
+        sampleRate: number;
+        ttsSampleRate: number;
+    } | null;
+};

package/dist/session-core-upload.d.ts

@@ -0,0 +1,16 @@
+import type { ClientMessage } from "@alexkroman1/aai/protocol";
+import type { ConnState, SessionSnapshot } from "./session-core-types.ts";
+/** Handle to the upload pipeline, owned by one session core. */
+export type UploadSender = {
+    /** See {@link SessionCore.sendAudioFile}. */
+    sendAudioFile(file: Blob): Promise<void>;
+    /** True while an upload is decoding or streaming — the mic must stay off. */
+    inFlight(): boolean;
+    /** Invalidate any in-flight upload (reset / close / reconnect). */
+    discard(): void;
+};
+export declare function createUploadSender(deps: {
+    conn: ConnState;
+    getSnapshot: () => SessionSnapshot;
+    sendJson: (msg: ClientMessage) => void;
+}): UploadSender;

package/dist/session-core.d.ts

@@ -1,71 +1,5 @@
-import type { AgentState, ChatMessage, SessionError, ToolCallInfo, VoiceSessionOptions } from "./types.ts";
-export type { AgentState, ChatMessage, SessionError, SessionErrorCode, ToolCallInfo, VoiceSessionOptions, WebSocketConstructor, } from "./types.ts";
-/**
- * A custom event emitted by the agent via `ctx.send`.
- *
- * @public
- */
-export type CustomEvent = {
-    readonly id: number;
-    readonly event: string;
-    readonly data: unknown;
-};
-/**
- * Immutable snapshot of the session state.
- *
- * Consumers (e.g. React hooks via `useSyncExternalStore`) read this to render.
- * A new object reference is created on every state change.
- *
- * @public
- */
-export type SessionSnapshot = {
-    readonly state: AgentState;
-    readonly messages: ChatMessage[];
-    readonly toolCalls: ToolCallInfo[];
-    readonly customEvents: CustomEvent[];
-    readonly userTranscript: string | null;
-    readonly agentTranscript: string | null;
-    readonly error: SessionError | null;
-    readonly started: boolean;
-    readonly running: boolean;
-};
-/**
- * A framework-agnostic voice session that manages WebSocket communication,
- * audio capture/playback, and agent state transitions.
- *
- * Uses a subscribe/getSnapshot pattern (compatible with React's
- * `useSyncExternalStore`). Implements `Disposable` for resource cleanup.
- *
- * @public
- */
-export type SessionCore = {
-    /** Return the current immutable state snapshot. */
-    getSnapshot(): SessionSnapshot;
-    /** Subscribe to state changes. Returns an unsubscribe function. */
-    subscribe(callback: () => void): () => void;
-    /**
-     * Open a WebSocket connection to the server and begin audio capture.
-     * @param options - Optional. `signal` is an AbortSignal that, when aborted, disconnects the session.
-     */
-    connect(options?: {
-        signal?: AbortSignal;
-    }): void;
-    /** Cancel the current agent turn and discard in-flight TTS audio. */
-    cancel(): void;
-    /** Clear messages, transcript, and error state without disconnecting. */
-    resetState(): void;
-    /** Reset the session: clear state and reconnect. */
-    reset(): void;
-    /** Close the WebSocket and release all audio resources. */
-    disconnect(): void;
-    /** Start the session for the first time (sets `started` and `running`). */
-    start(): void;
-    /** Toggle between connected and disconnected states. */
-    toggle(): void;
-    /** Alias for `disconnect` for use with `using`. */
-    [Symbol.dispose](): void;
-};
-export type SessionCoreOptions = VoiceSessionOptions;
+import type { SessionCore, SessionCoreOptions } from "./session-core-types.ts";
+export type { CustomEvent, SessionCore, SessionCoreOptions, SessionSnapshot, } from "./session-core-types.ts";
 /**
  * Create a framework-agnostic voice session core that connects to an AAI
  * server via WebSocket.

package/dist/session-core.js

@@ -1,467 +1,2 @@
-import { WS_OPEN, errorMessage } from "@alexkroman1/aai";
-import { ServerMessageSchema, lenientParse } from "@alexkroman1/aai/protocol";
-//#region session-core.ts
-/**
-* Framework-agnostic voice session core.
-*
-* Manages WebSocket communication, audio capture/playback, and agent state
-* transitions using a subscribe/getSnapshot pattern compatible with React's
-* `useSyncExternalStore` and other external store consumers.
-*
-* No dependency on React, Preact, or any UI framework.
-*/
-/** Cap on `customEvents` retained in the session snapshot to avoid unbounded growth. */
-const MAX_CUSTOM_EVENTS = 200;
-/** Cap on `messages` retained in the session snapshot; mirrors host-side DEFAULT_MAX_HISTORY. */
-const MAX_MESSAGES = 200;
-/** Cap on pre-init audio chunks buffered while `voiceIO` is initializing. ~100 chunks at
-*  typical S2S chunk sizes is well over a second of audio — far longer than init takes
-*  in practice, but bounded against pathological cases (mic-permission stalls). */
-const MAX_PREINIT_AUDIO_CHUNKS = 100;
-function appendCapped(list, item, cap) {
-	const next = [...list, item];
-	return next.length > cap ? next.slice(-cap) : next;
-}
-/**
-* Initialize audio capture and playback after the server sends a ready config.
-*
-* Lifecycle: dynamically import audio modules -> request microphone access ->
-* register AudioWorklet processors -> create a `VoiceIO` instance -> send
-* `audio_ready` to the server -> transition state to `"listening"`.
-*
-* Uses the connection `generation` counter to detect if `connect()` was called
-* while awaiting async operations; if so, the stale VoiceIO is closed immediately
-* to prevent it from being assigned to a newer connection.
-*
-* On failure (e.g. microphone permission denied, WebSocket closed mid-setup),
-* sets the error state and transitions to `"disconnected"`.
-*/
-async function initAudioCapture(conn, msg, deps) {
-	if (conn.audioSetupInFlight) return;
-	conn.audioSetupInFlight = true;
-	const gen = conn.generation;
-	try {
-		const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await Promise.all([
-			import("./audio.js"),
-			import("./worklets/capture-processor.js").then((m) => m.default),
-			import("./worklets/playback-processor.js").then((m) => m.default)
-		]);
-		const io = await createVoiceIO({
-			sttSampleRate: msg.sampleRate,
-			ttsSampleRate: msg.ttsSampleRate,
-			captureWorkletSrc: captureWorklet,
-			playbackWorkletSrc: playbackWorklet,
-			onMicData: (pcm16) => {
-				try {
-					deps.sendAudio(new Uint8Array(pcm16));
-				} catch {
-					console.debug("[aai-ui] sendAudio dropped: connection closed");
-				}
-			}
-		});
-		if (conn.generation !== gen || !conn.ws || conn.ws.readyState !== WS_OPEN) {
-			io.close();
-			return;
-		}
-		conn.voiceIO = io;
-		if (conn.preInitAudio.length > 0) {
-			for (const chunk of conn.preInitAudio) io.enqueue(chunk.buffer);
-			conn.preInitAudio = [];
-		}
-		deps.sendJson({ type: "audio_ready" });
-		deps.updateState({ state: "listening" });
-	} catch (err) {
-		if (conn.generation !== gen || !conn.ws || conn.ws.readyState !== WS_OPEN) return;
-		deps.updateState({
-			state: "error",
-			error: {
-				code: "audio",
-				message: `Microphone access failed: ${errorMessage(err)}`
-			},
-			running: false
-		});
-	} finally {
-		conn.audioSetupInFlight = false;
-	}
-}
-function buildWsUrl(platformUrl, resume, sessionId) {
-	const wsUrl = new URL("websocket", platformUrl.endsWith("/") ? platformUrl : `${platformUrl}/`);
-	wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:";
-	if (sessionId) wsUrl.searchParams.set("sessionId", sessionId);
-	else if (resume) wsUrl.searchParams.set("resume", "1");
-	return wsUrl;
-}
-/**
-* Create a framework-agnostic voice session core that connects to an AAI
-* server via WebSocket.
-*
-* Uses a subscribe/getSnapshot pattern for state management, compatible with
-* React's `useSyncExternalStore` and other external store integrations.
-*
-* @param options - Session configuration including the platform server URL.
-* @returns A {@link SessionCore} handle for controlling the session.
-*
-* @public
-*/
-function createSessionCore(options) {
-	const WS = options.WebSocket ?? WebSocket;
-	let currentSnapshot = {
-		state: "disconnected",
-		messages: [],
-		toolCalls: [],
-		customEvents: [],
-		userTranscript: null,
-		agentTranscript: null,
-		error: null,
-		started: false,
-		running: false
-	};
-	const subscribers = /* @__PURE__ */ new Set();
-	function notify() {
-		for (const sub of subscribers) sub();
-	}
-	function updateState(partial) {
-		currentSnapshot = {
-			...currentSnapshot,
-			...partial
-		};
-		notify();
-	}
-	function getSnapshot() {
-		return currentSnapshot;
-	}
-	function subscribe(callback) {
-		subscribers.add(callback);
-		return () => {
-			subscribers.delete(callback);
-		};
-	}
-	const conn = {
-		ws: null,
-		voiceIO: null,
-		audioSetupInFlight: false,
-		generation: 0,
-		preInitAudio: []
-	};
-	let connectionController = null;
-	let hasConnected = false;
-	function cleanupAudio() {
-		conn.audioSetupInFlight = false;
-		conn.voiceIO?.close();
-		conn.voiceIO = null;
-		conn.preInitAudio = [];
-	}
-	function resetState() {
-		updateState({
-			messages: [],
-			toolCalls: [],
-			customEvents: [],
-			userTranscript: null,
-			agentTranscript: null,
-			error: null
-		});
-	}
-	function sendJson(msg) {
-		if (conn.ws && conn.ws.readyState === WS_OPEN) conn.ws.send(JSON.stringify(msg));
-	}
-	function sendAudio(bytes) {
-		if (conn.ws && conn.ws.readyState === WS_OPEN) conn.ws.send(bytes);
-	}
-	const audioDeps = {
-		sendJson,
-		sendAudio,
-		updateState
-	};
-	/** Incremented on each turn boundary -- stale async callbacks compare against this. */
-	let handlerGeneration = 0;
-	/** Monotonically increasing counter for custom events -- used by useEvent to deduplicate. */
-	let customEventSeq = 0;
-	function appendCustomEvent(name, data) {
-		updateState({ customEvents: appendCapped(currentSnapshot.customEvents, {
-			id: ++customEventSeq,
-			event: name,
-			data
-		}, MAX_CUSTOM_EVENTS) });
-	}
-	function handleUserTranscriptEvent(text) {
-		handlerGeneration++;
-		updateState({
-			userTranscript: null,
-			messages: appendCapped(currentSnapshot.messages, {
-				role: "user",
-				content: text
-			}, MAX_MESSAGES),
-			state: "thinking"
-		});
-	}
-	function handleAgentTranscriptEvent(text) {
-		updateState({
-			agentTranscript: null,
-			messages: appendCapped(currentSnapshot.messages, {
-				role: "assistant",
-				content: text
-			}, MAX_MESSAGES)
-		});
-	}
-	/** Single entry point for all server->client session events. */
-	function handleEvent(e) {
-		if (currentSnapshot.state === "error" && e.type !== "error") updateState({
-			state: "disconnected",
-			error: null
-		});
-		switch (e.type) {
-			case "speech_started":
-				updateState({ userTranscript: "" });
-				break;
-			case "speech_stopped": break;
-			case "user_transcript":
-				handleUserTranscriptEvent(e.text);
-				break;
-			case "agent_transcript":
-				handleAgentTranscriptEvent(e.text);
-				break;
-			case "tool_call":
-				updateState({ toolCalls: [...currentSnapshot.toolCalls, {
-					callId: e.toolCallId,
-					name: e.toolName,
-					args: e.args ?? {},
-					status: "pending",
-					afterMessageIndex: currentSnapshot.messages.length - 1
-				}] });
-				break;
-			case "tool_call_done": {
-				const tcs = currentSnapshot.toolCalls;
-				const idx = tcs.findIndex((tc) => tc.callId === e.toolCallId);
-				if (idx !== -1) {
-					const updated = [...tcs];
-					const existing = updated[idx];
-					if (existing) updated[idx] = {
-						...existing,
-						status: "done",
-						result: e.result
-					};
-					updateState({ toolCalls: updated });
-				}
-				break;
-			}
-			case "reply_done":
-				updateState({ state: "listening" });
-				break;
-			case "cancelled":
-				handlerGeneration++;
-				conn.voiceIO?.flush();
-				updateState({
-					userTranscript: null,
-					agentTranscript: null,
-					state: "listening"
-				});
-				break;
-			case "reset":
-				handlerGeneration++;
-				conn.voiceIO?.flush();
-				updateState({
-					messages: [],
-					toolCalls: [],
-					customEvents: [],
-					userTranscript: null,
-					agentTranscript: null,
-					error: null,
-					state: "listening"
-				});
-				break;
-			case "custom_event":
-				appendCustomEvent(e.event, e.data);
-				break;
-			case "error":
-				console.error("Agent error:", e.message);
-				updateState({
-					state: "error",
-					error: {
-						code: e.code,
-						message: e.message
-					},
-					running: false
-				});
-				break;
-			case "idle_timeout": break;
-			default: break;
-		}
-	}
-	/** Enqueue a PCM16 audio chunk for playback. Transitions state to `"speaking"` on the first chunk. */
-	function playAudioChunk(chunk) {
-		if (currentSnapshot.state === "disconnected" && currentSnapshot.error !== null) return;
-		if (currentSnapshot.state !== "speaking") updateState({ state: "speaking" });
-		if (conn.voiceIO) conn.voiceIO.enqueue(chunk.buffer);
-		else if (conn.preInitAudio.length < MAX_PREINIT_AUDIO_CHUNKS) conn.preInitAudio.push(chunk);
-	}
-	/**
-	* Signal that the server has finished sending audio for this turn.
-	* Waits for the audio queue to drain, then transitions state to `"listening"`.
-	* Uses the `handlerGeneration` counter to discard stale completions from interrupted turns.
-	*/
-	function playAudioDone() {
-		const gen = handlerGeneration;
-		const io = conn.voiceIO;
-		if (io) io.done().then(() => {
-			if (handlerGeneration !== gen) return;
-			updateState({ state: "listening" });
-		}).catch((err) => {
-			console.warn("Audio playback done failed:", err);
-		});
-		else updateState({ state: "listening" });
-	}
-	/**
-	* Dispatch an incoming WebSocket message.
-	*
-	* Binary frames carry raw PCM16 audio chunks. Text frames are JSON-encoded
-	* {@link ServerMessage} values validated via Zod.
-	*
-	* Returns the parsed config if the message is a `config` message,
-	* otherwise `undefined`.
-	*/
-	function handleMessage(data) {
-		if (data instanceof ArrayBuffer) {
-			playAudioChunk(new Uint8Array(data));
-			return;
-		}
-		if (typeof data !== "string") {
-			console.warn("session-core: non-string, non-binary frame received; dropping");
-			return;
-		}
-		let raw;
-		try {
-			raw = JSON.parse(data);
-		} catch {
-			console.warn("session-core: invalid JSON; dropping");
-			return;
-		}
-		const parsed = lenientParse(ServerMessageSchema, raw);
-		if (!parsed.ok) {
-			if (parsed.malformed) console.warn("session-core: malformed server message", parsed.error);
-			return;
-		}
-		const msg = parsed.data;
-		if (msg.type === "config") return {
-			sampleRate: msg.sampleRate,
-			ttsSampleRate: msg.ttsSampleRate,
-			sid: msg.sessionId
-		};
-		if (msg.type === "audio_done") {
-			playAudioDone();
-			return;
-		}
-		handleEvent(msg);
-	}
-	function connect(opts) {
-		updateState({
-			state: "connecting",
-			error: null
-		});
-		connectionController?.abort();
-		cleanupAudio();
-		conn.ws?.close();
-		conn.ws = null;
-		conn.generation++;
-		const controller = new AbortController();
-		connectionController = controller;
-		const { signal: sig } = controller;
-		if (opts?.signal) opts.signal.addEventListener("abort", () => disconnect(), { signal: sig });
-		const resumeId = !hasConnected ? options.resumeSessionId : void 0;
-		const socket = new WS(buildWsUrl(options.platformUrl, hasConnected, resumeId).toString());
-		socket.binaryType = "arraybuffer";
-		conn.ws = socket;
-		socket.addEventListener("open", () => {
-			updateState({ state: "ready" });
-		}, { signal: sig });
-		socket.addEventListener("message", (event) => {
-			const config = handleMessage(event.data);
-			if (config) {
-				if (config.sid) options.onSessionId?.(config.sid);
-				const isReconnect = hasConnected;
-				hasConnected = true;
-				initAudioCapture(conn, config, audioDeps).catch((err) => {
-					audioDeps.updateState({
-						state: "error",
-						error: {
-							code: "audio",
-							message: `Audio capture failed: ${errorMessage(err)}`
-						},
-						running: false
-					});
-				});
-				if (isReconnect && currentSnapshot.messages.length > 0) sendJson({
-					type: "history",
-					messages: currentSnapshot.messages.map((m) => ({
-						role: m.role,
-						content: m.content
-					}))
-				});
-			}
-		}, { signal: sig });
-		socket.addEventListener("close", () => {
-			if (sig.aborted) return;
-			controller.abort();
-			cleanupAudio();
-			updateState({
-				state: "disconnected",
-				running: false
-			});
-		}, { signal: sig });
-	}
-	function cancel() {
-		conn.voiceIO?.flush();
-		updateState({ state: "listening" });
-		sendJson({ type: "cancel" });
-	}
-	function reset() {
-		conn.voiceIO?.flush();
-		if (conn.ws && conn.ws.readyState === WS_OPEN) {
-			sendJson({ type: "reset" });
-			return;
-		}
-		resetState();
-		disconnect();
-		connect();
-	}
-	function disconnect() {
-		connectionController?.abort();
-		connectionController = null;
-		cleanupAudio();
-		conn.ws?.close();
-		conn.ws = null;
-		updateState({
-			state: "disconnected",
-			running: false
-		});
-	}
-	function start() {
-		updateState({
-			started: true,
-			running: true
-		});
-		connect();
-	}
-	function toggle() {
-		if (currentSnapshot.running) disconnect();
-		else {
-			updateState({ running: true });
-			connect();
-		}
-	}
-	return {
-		getSnapshot,
-		subscribe,
-		connect,
-		cancel,
-		resetState,
-		reset,
-		disconnect,
-		start,
-		toggle,
-		[Symbol.dispose]() {
-			disconnect();
-		}
-	};
-}
-//#endregion
+import { t as createSessionCore } from "./session-core-D3NDaySY.js";
 export { createSessionCore };