@alexkroman1/aai 0.3.0

package/ui/signals.ts

@@ -0,0 +1,111 @@
+// Copyright 2025 the AAI authors. MIT license.
+
+import { batch, effect, type Signal, signal } from "@preact/signals";
+import type * as preact from "preact";
+import type { ComponentChildren } from "preact";
+import { createContext, h } from "preact";
+import { useContext } from "preact/hooks";
+import type { VoiceSession } from "./session.ts";
+
+/**
+ * Reactive session controls wrapping a {@linkcode VoiceSession} with Preact signals.
+ *
+ * Components access reactive data via `session` (e.g. `session.state`,
+ * `session.messages`). UI-only state (`started`, `running`) and actions
+ * (`start`, `toggle`, `reset`) live directly on this object.
+ */
+export type SessionSignals = {
+  /** The underlying voice session — all reactive data lives here. */
+  session: VoiceSession;
+  /** Whether the session has been started by the user. */
+  started: Signal<boolean>;
+  /** Whether the session is currently running (connected or connecting). */
+  running: Signal<boolean>;
+  /** Dispose the reactive effect that tracks error state. */
+  dispose(): void;
+  /** Start the session for the first time (sets `started` and `running`). */
+  start(): void;
+  /** Toggle between connected and disconnected states. */
+  toggle(): void;
+  /** Reset the session: clear state and reconnect. */
+  reset(): void;
+  /** Alias for {@linkcode dispose} for use with `using`. */
+  [Symbol.dispose](): void;
+};
+
+/**
+ * Wrap a {@linkcode VoiceSession} in Preact signals for reactive UI binding.
+ *
+ * Creates higher-level controls (start, toggle, reset) on top of the raw
+ * session, and automatically sets `running` to `false` when the session
+ * enters an error state.
+ *
+ * @param session - The voice session to wrap.
+ * @returns A {@linkcode SessionSignals} object for use in Preact components.
+ */
+export function createSessionControls(session: VoiceSession): SessionSignals {
+  const started = signal(false);
+  const running = signal(true);
+
+  const dispose = effect(() => {
+    if (session.state.value === "error") running.value = false;
+  });
+
+  return {
+    session,
+    started,
+    running,
+    dispose,
+    start() {
+      batch(() => {
+        started.value = true;
+        running.value = true;
+      });
+      session.connect();
+    },
+    toggle() {
+      if (running.value) session.disconnect();
+      else session.connect();
+      running.value = !running.value;
+    },
+    reset() {
+      session.reset();
+    },
+    [Symbol.dispose]() {
+      dispose();
+    },
+  };
+}
+
+const Ctx = createContext<SessionSignals | null>(null);
+
+/**
+ * Preact context provider that makes session signals available to descendant
+ * components via {@linkcode useSession}.
+ *
+ * @param props - Provider props.
+ * @param props.value - The session signals to provide.
+ * @param props.children - Child components that may consume the context.
+ * @returns A Preact VNode wrapping children in the session context.
+ */
+export function SessionProvider({
+  value,
+  children,
+}: {
+  value: SessionSignals;
+  children: ComponentChildren;
+}): preact.JSX.Element {
+  return h(Ctx.Provider, { value }, children);
+}
+
+/**
+ * Hook to access session signals from within a {@linkcode SessionProvider}.
+ *
+ * @returns The {@linkcode SessionSignals} from the nearest provider.
+ * @throws {Error} If called outside of a `SessionProvider`.
+ */
+export function useSession(): SessionSignals {
+  const ctx = useContext(Ctx);
+  if (!ctx) throw new Error("Hook useSession() requires a SessionProvider");
+  return ctx;
+}

package/ui/types.ts

@@ -0,0 +1,50 @@
+// Copyright 2025 the AAI authors. MIT license.
+/** Microphone buffer duration in seconds before sending to the server. */
+export const MIC_BUFFER_SECONDS = 0.1;
+
+/** Current state of the voice agent session. */
+export type AgentState =
+  | "disconnected"
+  | "connecting"
+  | "ready"
+  | "listening"
+  | "thinking"
+  | "speaking"
+  | "error";
+
+/** A chat message exchanged between user and assistant. */
+export type Message = {
+  /** The sender of the message. */
+  role: "user" | "assistant";
+  /** The text content of the message. */
+  text: string;
+};
+
+/** Info about a tool call for display in the UI. */
+export type ToolCallInfo = {
+  toolCallId: string;
+  toolName: string;
+  args: Record<string, unknown>;
+  status: "pending" | "done";
+  result?: string | undefined;
+  /** Index in the messages array where this tool call should appear. */
+  afterMessageIndex: number;
+};
+
+import type { SessionErrorCode } from "../sdk/protocol.ts";
+
+export type { SessionErrorCode };
+
+/** Error reported by the voice session. */
+export type SessionError = {
+  /** The category of the error. */
+  readonly code: SessionErrorCode;
+  /** A human-readable description of the error. */
+  readonly message: string;
+};
+
+/** Options for creating a voice session. */
+export type SessionOptions = {
+  /** Base URL of the AAI platform server. */
+  platformUrl: string;
+};

package/ui/worklets/capture-processor.js

@@ -0,0 +1,62 @@
+// Capture worklet: captures mic Float32 samples, resamples to STT rate if
+// needed, converts to Int16 PCM, and sends chunks to the main thread.
+
+const CaptureProcessorWorklet = `
+class CaptureProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.recording = false;
+    const opts = options.processorOptions || {};
+    this.fromRate = opts.contextRate || sampleRate;
+    this.toRate = opts.sttSampleRate || sampleRate;
+    this.ratio = this.fromRate / this.toRate;
+    this.needsResample = this.fromRate !== this.toRate;
+    this.port.onmessage = (e) => {
+      if (e.data.event === 'start') this.recording = true;
+      else if (e.data.event === 'stop') this.recording = false;
+    };
+  }
+
+  resample(input) {
+    const ratio = this.ratio;
+    const outLen = Math.ceil(input.length / ratio);
+    const out = new Float32Array(outLen);
+    for (let i = 0; i < outLen; i++) {
+      const srcIdx = i * ratio;
+      const idx = srcIdx | 0;
+      const frac = srcIdx - idx;
+      const a = input[idx];
+      const b = idx + 1 < input.length ? input[idx + 1] : a;
+      out[i] = a + frac * (b - a);
+    }
+    return out;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || !input[0] || !this.recording) return true;
+
+    const raw = input[0];
+    const samples = this.needsResample ? this.resample(raw) : raw;
+
+    // Convert Float32 -> Int16
+    const buffer = new ArrayBuffer(samples.length * 2);
+    const view = new DataView(buffer);
+    for (let i = 0; i < samples.length; i++) {
+      const s = Math.max(-1, Math.min(1, samples[i]));
+      view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+    }
+
+    this.port.postMessage({ event: 'chunk', buffer }, [buffer]);
+    return true;
+  }
+}
+
+registerProcessor('capture-processor', CaptureProcessor);
+`;
+
+const script = new Blob([CaptureProcessorWorklet], {
+  type: "application/javascript",
+});
+const src = URL.createObjectURL(script);
+export default src;

package/ui/worklets/playback-processor.js

@@ -0,0 +1,110 @@
+// Playback worklet: receives raw PCM16 LE bytes, handles byte alignment,
+// converts to float32, and plays with a small jitter buffer.
+
+const PlaybackProcessorWorklet = `
+class PlaybackProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.interrupted = false;
+    this.isDone = false;
+    this.playing = false;
+    const rate = options.processorOptions?.sampleRate ?? 24000;
+    // Wait for ~400ms of audio before starting.
+    // If 'done' arrives first (short utterance), start immediately.
+    this.jitterSamples = Math.floor(rate * 0.4);
+    // Carry-over byte for split samples across chunks
+    this.carry = null;
+    // Float32 sample buffer — 60s at the context sample rate
+    this.samples = new Float32Array(rate * 60);
+    this.writePos = 0;
+    this.readPos = 0;
+    // Report playback position every ~50ms for word-level text sync
+    this.progressInterval = Math.floor(rate * 0.05);
+    this.samplesSinceProgress = 0;
+
+    this.port.onmessage = (e) => {
+      const d = e.data;
+      if (d.event === 'write') {
+        this.ingestBytes(d.buffer);
+      } else if (d.event === 'interrupt') {
+        this.interrupted = true;
+      } else if (d.event === 'done') {
+        this.isDone = true;
+      }
+    };
+  }
+
+  ingestBytes(uint8) {
+    let bytes = uint8;
+
+    if (this.carry !== null) {
+      const merged = new Uint8Array(1 + bytes.length);
+      merged[0] = this.carry;
+      merged.set(bytes, 1);
+      bytes = merged;
+      this.carry = null;
+    }
+
+    if (bytes.length % 2 !== 0) {
+      this.carry = bytes[bytes.length - 1];
+      bytes = bytes.subarray(0, bytes.length - 1);
+    }
+
+    if (bytes.length === 0) return;
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);
+    const numSamples = bytes.length / 2;
+    for (let i = 0; i < numSamples; i++) {
+      this.samples[this.writePos++] = view.getInt16(i * 2, true) / 0x8000;
+    }
+  }
+
+  process(inputs, outputs) {
+    const out = outputs[0][0];
+    if (this.interrupted) {
+      this.port.postMessage({ event: 'stop' });
+      return false;
+    }
+
+    const avail = this.writePos - this.readPos;
+
+    // Wait for jitter buffer to fill, unless done (short utterance)
+    if (!this.playing) {
+      if (avail >= this.jitterSamples || this.isDone) {
+        this.playing = true;
+      } else {
+        for (let i = 0; i < out.length; i++) out[i] = 0;
+        return true;
+      }
+    }
+
+    if (avail > 0) {
+      const n = Math.min(avail, out.length);
+      out.set(this.samples.subarray(this.readPos, this.readPos + n));
+      this.readPos += n;
+      this.samplesSinceProgress += n;
+      if (this.samplesSinceProgress >= this.progressInterval) {
+        this.port.postMessage({ event: 'progress', readPos: this.readPos });
+        this.samplesSinceProgress = 0;
+      }
+      for (let i = n; i < out.length; i++) out[i] = 0;
+      return true;
+    }
+
+    // No data: output silence, stop only when done
+    for (let i = 0; i < out.length; i++) out[i] = 0;
+    if (this.isDone) {
+      this.port.postMessage({ event: 'stop' });
+      return false;
+    }
+    return true;
+  }
+}
+
+registerProcessor('playback-processor', PlaybackProcessor);
+`;
+
+const script = new Blob([PlaybackProcessorWorklet], {
+  type: "application/javascript",
+});
+const src = URL.createObjectURL(script);
+export default src;