@alexkroman1/aai 0.3.0

package/ui/mount.tsx

@@ -0,0 +1,112 @@
+// Copyright 2025 the AAI authors. MIT license.
+
+import type { ComponentType } from "preact";
+import { render } from "preact";
+import { MountConfigProvider } from "./mount_context.ts";
+import { createVoiceSession, type VoiceSession } from "./session.ts";
+import { createSessionControls, SessionProvider, type SessionSignals } from "./signals.ts";
+
+/** Theme overrides for the default UI. Applied as CSS custom properties. */
+export type MountTheme = {
+  /** Background color. Default: `#101010`. */
+  bg?: string;
+  /** Primary accent color. Default: `#fab283`. */
+  primary?: string;
+  /** Main text color. */
+  text?: string;
+  /** Surface/card color. */
+  surface?: string;
+  /** Border color. */
+  border?: string;
+};
+
+/** Options for {@linkcode mount}. */
+export type MountOptions = {
+  /** CSS selector or DOM element to render into. Defaults to `"#app"`. */
+  target?: string | HTMLElement;
+  /** Base URL of the AAI platform server. Derived from `location.href` by default. */
+  platformUrl?: string;
+  /** Agent title shown in the header and start screen. */
+  title?: string;
+  /** Theme color overrides. */
+  theme?: MountTheme;
+};
+
+/**
+ * Handle returned by {@linkcode mount} for cleanup.
+ *
+ * Implements {@linkcode Disposable} so it can be used with `using`.
+ */
+export type MountHandle = {
+  /** The underlying voice session. */
+  session: VoiceSession;
+  /** Reactive session controls for the mounted UI. */
+  signals: SessionSignals;
+  /** Unmount the UI, remove injected styles, and disconnect the session. */
+  dispose(): void;
+  /** Alias for {@linkcode dispose} for use with `using`. */
+  [Symbol.dispose](): void;
+};
+
+function resolveContainer(target: string | HTMLElement = "#app"): HTMLElement {
+  const el = typeof target === "string" ? document.querySelector(target) : target;
+  if (!el) throw new Error(`Element not found: ${target}`);
+  return el as HTMLElement;
+}
+
+/**
+ * Mount a Preact component with voice session wiring.
+ *
+ * Creates a {@linkcode VoiceSession}, wraps it in
+ * {@linkcode SessionSignals}, and renders the component
+ * inside a {@linkcode SessionProvider}.
+ *
+ * @param Component - The Preact component to render.
+ * @param options - Mount options (target element, platform URL).
+ * @returns A {@linkcode MountHandle} for cleanup.
+ * @throws {Error} If the target element is not found in the DOM.
+ */
+export function mount(Component: ComponentType, options?: MountOptions): MountHandle {
+  const container = resolveContainer(options?.target);
+
+  const platformUrl =
+    options?.platformUrl ?? globalThis.location.origin + globalThis.location.pathname;
+  const session = createVoiceSession({ platformUrl });
+  const signals = createSessionControls(session);
+
+  const mountConfig = { title: options?.title, theme: options?.theme };
+
+  // Apply theme overrides as CSS custom properties on the container.
+  if (options?.theme) {
+    const t = options.theme;
+    const el = container as HTMLElement;
+    if (t.bg) el.style.setProperty("--color-aai-bg", t.bg);
+    if (t.primary) el.style.setProperty("--color-aai-primary", t.primary);
+    if (t.text) el.style.setProperty("--color-aai-text", t.text);
+    if (t.surface) el.style.setProperty("--color-aai-surface", t.surface);
+    if (t.border) el.style.setProperty("--color-aai-border", t.border);
+  }
+
+  render(
+    <MountConfigProvider value={mountConfig}>
+      <SessionProvider value={signals}>
+        <Component />
+      </SessionProvider>
+    </MountConfigProvider>,
+    container,
+  );
+
+  const handle: MountHandle = {
+    session,
+    signals,
+    dispose() {
+      render(null, container);
+      signals.dispose();
+      session.disconnect();
+    },
+    [Symbol.dispose]() {
+      handle.dispose();
+    },
+  };
+  return handle;
+}

package/ui/mount_context.ts

@@ -0,0 +1,19 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { createContext } from "preact";
+import { useContext } from "preact/hooks";
+import type { MountTheme } from "./mount.tsx";
+
+/** Resolved mount-level configuration available to default UI components. */
+export type MountConfig = {
+  title?: string | undefined;
+  theme?: MountTheme | undefined;
+};
+
+const Ctx = createContext<MountConfig>({});
+
+export const MountConfigProvider = Ctx.Provider;
+
+/** Read mount config (title, theme) from the nearest provider. */
+export function useMountConfig(): MountConfig {
+  return useContext(Ctx);
+}

package/ui/session.ts

@@ -0,0 +1,456 @@
+// Copyright 2025 the AAI authors. MIT license.
+
+import { batch, type Signal, signal } from "@preact/signals";
+import type { ClientEvent, ClientMessage, ReadyConfig, ServerMessage } from "../sdk/protocol.ts";
+import { PROTOCOL_VERSION } from "../sdk/protocol.ts";
+
+const SUPPORTED_PROTOCOL_VERSION = PROTOCOL_VERSION;
+
+import type { VoiceIO } from "./audio.ts";
+import type { AgentState, Message, SessionError, SessionOptions, ToolCallInfo } from "./types.ts";
+
+/**
+ * A reactive voice session that manages WebSocket communication,
+ * audio capture/playback, and agent state transitions.
+ *
+ * Uses plain JSON text frames and binary audio frames for communication
+ * and native WebSocket for the connection.
+ *
+ * Implements {@linkcode Disposable} for resource cleanup via `using`.
+ */
+export type VoiceSession = {
+  /** Current agent state (connecting, listening, thinking, etc.). */
+  readonly state: Signal<AgentState>;
+  /** Chat message history for the session. */
+  readonly messages: Signal<Message[]>;
+  /** Active tool calls for the current turn. */
+  readonly toolCalls: Signal<ToolCallInfo[]>;
+  /**
+   * Live user utterance from STT/VAD.
+   * `null` = not speaking, `""` = speech detected but no text yet,
+   * non-empty string = partial/final transcript text.
+   */
+  readonly userUtterance: Signal<string | null>;
+  /** Current session error, or `null` if no error. */
+  readonly error: Signal<SessionError | null>;
+  /** Disconnection info, or `null` if connected. */
+  readonly disconnected: Signal<{ intentional: boolean } | null>;
+  /**
+   * Open a WebSocket connection to the server and begin audio capture.
+   *
+   * @param options - Optional connection options.
+   * @param options.signal - An AbortSignal that, when aborted, disconnects the session.
+   */
+  connect(options?: { signal?: AbortSignal }): void;
+  /** Cancel the current agent turn and discard in-flight TTS audio. */
+  cancel(): void;
+  /** Clear messages, transcript, and error state without disconnecting. */
+  resetState(): void;
+  /** Reset the session: clear state and reconnect. */
+  reset(): void;
+  /** Close the WebSocket and release all audio resources. */
+  disconnect(): void;
+  /** Alias for {@linkcode disconnect} for use with `using`. */
+  [Symbol.dispose](): void;
+};
+
+/**
+ * Handles server→client messages and updates reactive Preact signals
+ * accordingly (state transitions, transcripts, messages, audio playback).
+ */
+/** @internal Exported for testing only. */
+export class ClientHandler {
+  #state: Signal<AgentState>;
+  #messages: Signal<Message[]>;
+  #toolCalls: Signal<ToolCallInfo[]>;
+  #userUtterance: Signal<string | null>;
+  #error: Signal<SessionError | null>;
+  #voiceIO: () => VoiceIO | null;
+  /** Incremented on each turn boundary — stale async callbacks compare against this. */
+  #generation = 0;
+  constructor(opts: {
+    state: Signal<AgentState>;
+    messages: Signal<Message[]>;
+    toolCalls: Signal<ToolCallInfo[]>;
+    userUtterance: Signal<string | null>;
+    error: Signal<SessionError | null>;
+    voiceIO: () => VoiceIO | null;
+  }) {
+    this.#state = opts.state;
+    this.#messages = opts.messages;
+    this.#toolCalls = opts.toolCalls;
+    this.#userUtterance = opts.userUtterance;
+    this.#error = opts.error;
+    this.#voiceIO = opts.voiceIO;
+  }
+
+  /** Single entry point for all server→client session events. */
+  /** Sentinel value indicating speech detected but no transcript yet. */
+  static readonly speechActive = "\x00";
+
+  event(e: ClientEvent): void {
+    switch (e.type) {
+      case "speech_started":
+        this.#userUtterance.value = "";
+        break;
+      case "speech_stopped":
+        // VAD detected end of speech — processing will follow.
+        break;
+      case "transcript":
+        this.#userUtterance.value = e.text;
+        break;
+      case "turn":
+        this.#generation++;
+        batch(() => {
+          this.#userUtterance.value = null;
+          this.#messages.value = [...this.#messages.value, { role: "user", text: e.text }];
+          this.#state.value = "thinking";
+        });
+        break;
+      case "chat":
+        this.#messages.value = [...this.#messages.value, { role: "assistant", text: e.text }];
+        break;
+      case "tool_call_start":
+        this.#toolCalls.value = [
+          ...this.#toolCalls.value,
+          {
+            toolCallId: e.toolCallId,
+            toolName: e.toolName,
+            args: e.args,
+            status: "pending",
+            afterMessageIndex: this.#messages.value.length - 1,
+          },
+        ];
+        break;
+      case "tool_call_done": {
+        const tcs = this.#toolCalls.value;
+        const idx = tcs.findIndex((tc) => tc.toolCallId === e.toolCallId);
+        if (idx !== -1) {
+          const updated = [...tcs];
+          updated[idx] = { ...updated[idx]!, status: "done", result: e.result };
+          this.#toolCalls.value = updated;
+        }
+        break;
+      }
+      case "tts_done":
+        // No-audio turns (empty LLM result) still use this event
+        // to transition back to listening. Audio turns signal via stream end.
+        this.#state.value = "listening";
+        break;
+      case "cancelled":
+        this.#generation++;
+        this.#voiceIO()?.flush();
+        this.#userUtterance.value = null;
+        this.#state.value = "listening";
+        break;
+      case "reset": {
+        this.#generation++;
+        this.#voiceIO()?.flush();
+        batch(() => {
+          this.#messages.value = [];
+          this.#toolCalls.value = [];
+          this.#userUtterance.value = null;
+          this.#error.value = null;
+          this.#state.value = "listening";
+        });
+        break;
+      }
+      case "error":
+        console.error("Agent error:", e.message);
+        batch(() => {
+          this.#error.value = {
+            code: e.code,
+            message: e.message,
+          };
+          this.#state.value = "error";
+        });
+        break;
+    }
+  }
+
+  playAudioChunk(chunk: Uint8Array): void {
+    if (this.#state.value === "error") return;
+    if (this.#state.value !== "speaking") {
+      this.#state.value = "speaking";
+    }
+    this.#voiceIO()?.enqueue(chunk.buffer as ArrayBuffer);
+  }
+
+  playAudioDone(): void {
+    const gen = this.#generation;
+    const io = this.#voiceIO();
+    if (io) {
+      void io.done().then(() => {
+        if (this.#generation !== gen) return;
+        this.#state.value = "listening";
+      });
+    } else {
+      this.#state.value = "listening";
+    }
+  }
+
+  /**
+   * Dispatch an incoming WebSocket message (text or binary).
+   *
+   * Returns the parsed config if the message is a `config` message,
+   * otherwise `null`.
+   */
+  handleMessage(data: string | ArrayBuffer): ReadyConfig | null {
+    // Binary frame → raw PCM16 TTS audio
+    if (data instanceof ArrayBuffer) {
+      this.playAudioChunk(new Uint8Array(data));
+      return null;
+    }
+
+    // Text frame → JSON message
+    let msg: ServerMessage;
+    try {
+      msg = JSON.parse(data);
+    } catch {
+      return null;
+    }
+
+    if (msg.type === "config") {
+      const { type: _, ...config } = msg;
+      return config as ReadyConfig;
+    }
+
+    if (msg.type === "audio_done") {
+      this.playAudioDone();
+      return null;
+    }
+
+    // All other messages are ClientEvent
+    this.event(msg as ClientEvent);
+    return null;
+  }
+}
+
+/**
+ * Create a voice session that connects to an AAI server via WebSocket.
+ *
+ * Uses plain JSON text frames and binary audio frames for communication.
+ *
+ * @param options - Session configuration including the platform server URL.
+ * @returns A {@linkcode VoiceSession} handle for controlling the session.
+ */
+export function createVoiceSession(options: SessionOptions): VoiceSession {
+  const state = signal<AgentState>("disconnected");
+  const messages = signal<Message[]>([]);
+  const toolCalls = signal<ToolCallInfo[]>([]);
+  const userUtterance = signal<string | null>(null);
+  const error = signal<SessionError | null>(null);
+  const disconnected = signal<{ intentional: boolean } | null>(null);
+
+  let ws: WebSocket | null = null;
+  let voiceIO: VoiceIO | null = null;
+  let connectionController: AbortController | null = null;
+  let hasConnected = false;
+  let audioSetupInFlight = false;
+  function cleanupAudio(): void {
+    audioSetupInFlight = false;
+    void voiceIO?.close();
+    voiceIO = null;
+  }
+
+  function resetState(): void {
+    batch(() => {
+      messages.value = [];
+      toolCalls.value = [];
+      userUtterance.value = null;
+      error.value = null;
+    });
+  }
+
+  function send(msg: ClientMessage): void {
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify(msg));
+    }
+  }
+
+  function sendBinary(data: ArrayBuffer): void {
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(data);
+    }
+  }
+
+  async function handleReady(msg: ReadyConfig): Promise<void> {
+    if (audioSetupInFlight) return;
+
+    // Protocol version check
+    if (msg.protocolVersion !== SUPPORTED_PROTOCOL_VERSION) {
+      batch(() => {
+        error.value = {
+          code: "protocol",
+          message: `Server protocol v${msg.protocolVersion} is not compatible with client v${SUPPORTED_PROTOCOL_VERSION}. Please redeploy your agent.`,
+        };
+        state.value = "error";
+      });
+      return;
+    }
+
+    audioSetupInFlight = true;
+    try {
+      const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await Promise.all([
+        import("./audio.ts"),
+        import("./worklets/capture-processor.js").then((m) => m.default as unknown as string),
+        import("./worklets/playback-processor.js").then((m) => m.default as unknown as string),
+      ]);
+      const io = await createVoiceIO({
+        sttSampleRate: msg.sampleRate,
+        ttsSampleRate: msg.ttsSampleRate,
+        captureWorkletSrc: captureWorklet,
+        playbackWorkletSrc: playbackWorklet,
+        onMicData: (pcm16: ArrayBuffer) => {
+          // Always stream audio — S2S handles VAD natively.
+          try {
+            sendBinary(pcm16);
+          } catch {
+            /* connection may be closed */
+          }
+        },
+      });
+      if (!ws || ws.readyState !== WebSocket.OPEN) {
+        io.close();
+        return;
+      }
+      voiceIO = io;
+      send({ type: "audio_ready" });
+      state.value = "listening";
+    } catch (err: unknown) {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      batch(() => {
+        error.value = {
+          code: "audio",
+          message: `Microphone access failed: ${err instanceof Error ? err.message : String(err)}`,
+        };
+        state.value = "error";
+      });
+    } finally {
+      audioSetupInFlight = false;
+    }
+  }
+
+  function connect(opts?: { signal?: AbortSignal }): void {
+    disconnected.value = null;
+    state.value = "connecting";
+    connectionController?.abort();
+    const controller = new AbortController();
+    connectionController = controller;
+    const { signal: sig } = controller;
+
+    if (opts?.signal) {
+      opts.signal.addEventListener("abort", () => disconnect(), {
+        signal: sig,
+      });
+    }
+
+    const base = options.platformUrl;
+    const wsUrl = new URL("websocket", base.endsWith("/") ? base : `${base}/`);
+    wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:";
+    if (hasConnected) wsUrl.searchParams.set("resume", "1");
+
+    const socket = new WebSocket(wsUrl.toString());
+    socket.binaryType = "arraybuffer";
+    ws = socket;
+
+    const handler = new ClientHandler({
+      state,
+      messages,
+      toolCalls,
+      userUtterance,
+      error,
+      voiceIO: () => voiceIO,
+    });
+
+    socket.addEventListener(
+      "open",
+      () => {
+        state.value = "ready";
+      },
+      { signal: sig },
+    );
+
+    socket.addEventListener(
+      "message",
+      (event: Event) => {
+        const msgEvent = event as MessageEvent;
+        const config = handler.handleMessage(msgEvent.data);
+        if (config) {
+          hasConnected = true;
+          void handleReady(config);
+
+          // Send history if reconnecting
+          if (hasConnected && messages.value.length > 0) {
+            send({
+              type: "history",
+              messages: messages.value.map((m) => ({
+                role: m.role,
+                text: m.text,
+              })),
+            });
+          }
+        }
+      },
+      { signal: sig },
+    );
+
+    socket.addEventListener(
+      "close",
+      () => {
+        if (sig.aborted) {
+          return;
+        }
+        controller.abort();
+        disconnected.value = { intentional: false };
+        cleanupAudio();
+        state.value = "disconnected";
+      },
+      { signal: sig },
+    );
+  }
+
+  function cancel(): void {
+    voiceIO?.flush();
+    state.value = "listening";
+    send({ type: "cancel" });
+  }
+
+  function reset(): void {
+    voiceIO?.flush();
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      send({ type: "reset" });
+      return;
+    }
+    resetState();
+    disconnect();
+    connect();
+  }
+
+  function disconnect(): void {
+    connectionController?.abort();
+    connectionController = null;
+    cleanupAudio();
+    ws?.close();
+    ws = null;
+    state.value = "disconnected";
+    disconnected.value = { intentional: true };
+  }
+
+  return {
+    state,
+    messages,
+    toolCalls,
+    userUtterance,
+    error,
+    disconnected,
+    connect,
+    cancel,
+    resetState,
+    reset,
+    disconnect,
+    [Symbol.dispose]() {
+      disconnect();
+    },
+  };
+}

package/ui/session_mod.ts

@@ -0,0 +1,27 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * WebSocket session management for AAI voice agents.
+ *
+ * Provides the low-level voice session without any Preact/UI dependencies.
+ *
+ * @example
+ * ```ts
+ * import { createVoiceSession } from "@aai/ui/session";
+ *
+ * const session = createVoiceSession({ platformUrl: "https://example.com" });
+ * session.connect();
+ * ```
+ *
+ * @module
+ */
+
+export type { VoiceSession } from "./session.ts";
+export { createVoiceSession } from "./session.ts";
+export type {
+  AgentState,
+  Message,
+  SessionError,
+  SessionErrorCode,
+  SessionOptions,
+  ToolCallInfo,
+} from "./types.ts";