@alexkroman1/aai 1.4.5 → 1.5.0

package/host/providers/stt/deepgram.test.ts

@@ -0,0 +1,229 @@
+// Copyright 2026 the AAI authors. MIT license.
+/** Unit test for the Deepgram STT adapter (mocked SDK). */
+
+import { describe, expect, test, vi } from "vitest";
+import { flush } from "../../_test-utils.ts";
+import { type DeepgramSession, openDeepgram } from "./deepgram.ts";
+
+// ---------------------------------------------------------------------------
+// Mock the `@deepgram/sdk` so no real sockets are opened.
+//
+// Each fake `V1Socket` keeps one listener per event (matching the real SDK's
+// `on()` which replaces rather than appends) and exposes `_fire(event, data)`
+// for tests to inject events. The adapter's `open()` returns a
+// `DeepgramSession` with a `_connection` pointer (which in tests is the fake)
+// giving the test a handle to `_fire`.
+// ---------------------------------------------------------------------------
+
+interface FakeSocket {
+  on(ev: string, fn: (...args: unknown[]) => void): void;
+  connect(): FakeSocket;
+  waitForOpen(): Promise<void>;
+  close(): void;
+  sendMedia(_data: ArrayBufferView): void;
+  _fire(ev: string, ...args: unknown[]): void;
+}
+
+vi.mock("@deepgram/sdk", () => {
+  const makeFakeSocket = (): FakeSocket => {
+    const listeners = new Map<string, (...args: unknown[]) => void>();
+    const fake: FakeSocket = {
+      on(ev, fn) {
+        // V1Socket replaces — not appends — the listener per event.
+        listeners.set(ev, fn);
+      },
+      connect() {
+        return fake;
+      },
+      async waitForOpen() {
+        // Immediately resolves in tests.
+      },
+      close() {
+        /* no-op */
+      },
+      sendMedia(_data: ArrayBufferView) {
+        /* no-op */
+      },
+      _fire(ev, ...args) {
+        const fn = listeners.get(ev);
+        if (fn) fn(...args);
+      },
+    };
+    return fake;
+  };
+
+  return {
+    DeepgramClient: class {
+      listen = {
+        v1: {
+          connect: (_args: unknown): Promise<FakeSocket> => Promise.resolve(makeFakeSocket()),
+        },
+      };
+    },
+  };
+});
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeResult(transcript: string, isFinal: boolean) {
+  return {
+    type: "Results" as const,
+    channel_index: [0],
+    duration: 1,
+    start: 0,
+    is_final: isFinal,
+    channel: { alternatives: [{ transcript, confidence: 0.9, words: [] }] },
+    metadata: { request_id: "mock" },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("Deepgram STT adapter", () => {
+  test("openDeepgram({}) returns an opener with name 'deepgram'", () => {
+    const opener = openDeepgram({});
+    expect(opener.name).toBe("deepgram");
+  });
+
+  test("throws stt_auth_failed when API key is missing", async () => {
+    // Clear env var for this test.
+    const saved = process.env.DEEPGRAM_API_KEY;
+    delete process.env.DEEPGRAM_API_KEY;
+
+    const opener = openDeepgram({});
+    const controller = new AbortController();
+
+    await expect(
+      opener.open({ sampleRate: 16_000, apiKey: "", signal: controller.signal }),
+    ).rejects.toMatchObject({ code: "stt_auth_failed" });
+
+    process.env.DEEPGRAM_API_KEY = saved;
+  });
+
+  test("final transcript fires 'final' event with text", async () => {
+    const opener = openDeepgram({ model: "nova-3" });
+    const controller = new AbortController();
+    const session = (await opener.open({
+      sampleRate: 16_000,
+      apiKey: "test-key",
+      signal: controller.signal,
+    })) as DeepgramSession;
+
+    const finals: string[] = [];
+    session.on("final", (t) => finals.push(t));
+
+    const fake = session._connection as unknown as FakeSocket;
+    fake._fire("message", makeResult("hello world", true));
+
+    await flush();
+    expect(finals).toEqual(["hello world"]);
+
+    await session.close();
+  });
+
+  test("interim transcript fires 'partial' event with text", async () => {
+    const opener = openDeepgram({ model: "nova-3" });
+    const controller = new AbortController();
+    const session = (await opener.open({
+      sampleRate: 16_000,
+      apiKey: "test-key",
+      signal: controller.signal,
+    })) as DeepgramSession;
+
+    const partials: string[] = [];
+    session.on("partial", (t) => partials.push(t));
+
+    const fake = session._connection as unknown as FakeSocket;
+    fake._fire("message", makeResult("hel", false));
+    fake._fire("message", makeResult("hello", false));
+
+    await flush();
+    expect(partials).toEqual(["hel", "hello"]);
+
+    await session.close();
+  });
+
+  test("empty transcript is NOT emitted (neither partial nor final)", async () => {
+    const opener = openDeepgram({});
+    const controller = new AbortController();
+    const session = (await opener.open({
+      sampleRate: 16_000,
+      apiKey: "test-key",
+      signal: controller.signal,
+    })) as DeepgramSession;
+
+    const partials: string[] = [];
+    const finals: string[] = [];
+    session.on("partial", (t) => partials.push(t));
+    session.on("final", (t) => finals.push(t));
+
+    const fake = session._connection as unknown as FakeSocket;
+    // Fire results with empty transcript — neither should be emitted.
+    fake._fire("message", makeResult("", false));
+    fake._fire("message", makeResult("", true));
+
+    await flush();
+    expect(partials).toEqual([]);
+    expect(finals).toEqual([]);
+
+    await session.close();
+  });
+
+  test("close fires close() and subsequent events are ignored (no double-close crash)", async () => {
+    const opener = openDeepgram({});
+    const controller = new AbortController();
+    const session = (await opener.open({
+      sampleRate: 16_000,
+      apiKey: "test-key",
+      signal: controller.signal,
+    })) as DeepgramSession;
+
+    const finals: string[] = [];
+    session.on("final", (t) => finals.push(t));
+
+    await session.close();
+
+    // Subsequent close should not throw.
+    await session.close();
+
+    // Events after close should be dropped.
+    const fake = session._connection as unknown as FakeSocket;
+    fake._fire("message", makeResult("should be ignored", true));
+
+    await flush();
+    expect(finals).toEqual([]);
+  });
+
+  test("sendAudio(Int16Array) forwards PCM bytes to the connection", async () => {
+    const opener = openDeepgram({});
+    const controller = new AbortController();
+    const session = (await opener.open({
+      sampleRate: 16_000,
+      apiKey: "test-key",
+      signal: controller.signal,
+    })) as DeepgramSession;
+
+    const fake = session._connection as unknown as FakeSocket;
+    const sent: ArrayBufferView[] = [];
+    fake.sendMedia = (data: ArrayBufferView) => sent.push(data);
+
+    const pcm = new Int16Array([100, 200, 300]);
+    session.sendAudio(pcm);
+
+    expect(sent).toHaveLength(1);
+    // The sent buffer should contain the same bytes as the Int16Array.
+    const sentBytes = new Uint8Array(
+      (sent[0] as Uint8Array).buffer,
+      (sent[0] as Uint8Array).byteOffset,
+      (sent[0] as Uint8Array).byteLength,
+    );
+    const expectedBytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+    expect(sentBytes).toEqual(expectedBytes);
+
+    await session.close();
+  });
+});

package/host/providers/stt/deepgram.ts

@@ -0,0 +1,172 @@
+// Copyright 2026 the AAI authors. MIT license.
+/**
+ * Deepgram Nova streaming STT opener (host-only).
+ *
+ * The user-facing descriptor factory (`deepgram(...)`) lives in
+ * `sdk/providers/stt/deepgram.ts`. This module is the host-side
+ * counterpart: it takes the descriptor options + an API key and
+ * returns an {@link SttOpener} that the pipeline session drives.
+ *
+ * Default model: `"nova-3"`. Any string is forwarded verbatim to the SDK.
+ *
+ * This adapter targets the Deepgram SDK v5 (`@deepgram/sdk@^5`). The v5
+ * streaming API is:
+ *   `client.listen.v1.connect(args)` → `Promise<V1Socket>`
+ * followed by:
+ *   `socket.connect()` + `socket.waitForOpen()` to establish the connection.
+ */
+
+import { DeepgramClient, type listen } from "@deepgram/sdk";
+import { createNanoEvents, type Emitter } from "nanoevents";
+import type { DeepgramOptions } from "../../../sdk/providers/stt/deepgram.ts";
+import {
+  makeSttError,
+  type SttEvents,
+  type SttOpener,
+  type SttOpenOptions,
+  type SttSession,
+} from "../../../sdk/providers.ts";
+
+// V1Socket type from the Deepgram SDK (accessed through the listen namespace).
+type V1Socket = Awaited<ReturnType<InstanceType<typeof DeepgramClient>["listen"]["v1"]["connect"]>>;
+
+/** Internal: SttSession with a test-only handle to the raw SDK socket. */
+export interface DeepgramSession extends SttSession {
+  /** @internal Test-only: exposes the underlying SDK socket for fixture replay. */
+  readonly _connection: V1Socket;
+}
+
+type MessagePayload =
+  | listen.ListenV1Results
+  | listen.ListenV1Metadata
+  | listen.ListenV1UtteranceEnd
+  | listen.ListenV1SpeechStarted;
+
+/**
+ * Handle an incoming Deepgram transcript message, emitting `partial` or
+ * `final` events on the emitter. Empty transcripts are silently dropped.
+ */
+function handleMessage(data: MessagePayload, closed: boolean, emitter: Emitter<SttEvents>): void {
+  if (closed) return;
+  if (data.type !== "Results") return;
+  const result = data as listen.ListenV1Results;
+  const text = result.channel?.alternatives?.[0]?.transcript ?? "";
+  if (result.is_final) {
+    if (text.length > 0) emitter.emit("final", text);
+  } else if (text.length > 0) {
+    emitter.emit("partial", text);
+  }
+}
+
+/** Wire Deepgram socket events onto the nanoevents emitter. */
+function wireSocketEvents(
+  connection: V1Socket,
+  emitter: Emitter<SttEvents>,
+  getIsClosed: () => boolean,
+): void {
+  connection.on("message", (data: MessagePayload) => handleMessage(data, getIsClosed(), emitter));
+  connection.on("error", (err: Error) => {
+    if (getIsClosed()) return;
+    emitter.emit("error", makeSttError("stt_stream_error", err?.message ?? String(err)));
+  });
+  connection.on("close", (event: { code?: number }) => {
+    if (getIsClosed()) return;
+    const code = event?.code;
+    // 1000 = normal closure.
+    if (code !== undefined && code !== 1000) {
+      emitter.emit("error", makeSttError("stt_stream_error", `socket closed ${code}`));
+    }
+  });
+}
+
+/** Wire the AbortSignal to the close function. */
+function wireAbortSignal(signal: AbortSignal, close: () => Promise<void>): void {
+  if (signal.aborted) {
+    void close();
+  } else {
+    signal.addEventListener("abort", () => void close(), { once: true });
+  }
+}
+
+/** Build an {@link SttOpener} from resolved Deepgram descriptor options. */
+export function openDeepgram(opts: DeepgramOptions = {}): SttOpener {
+  return {
+    name: "deepgram",
+    async open(openOpts: SttOpenOptions): Promise<SttSession> {
+      const apiKey = openOpts.apiKey || process.env.DEEPGRAM_API_KEY;
+      if (!apiKey) {
+        throw makeSttError(
+          "stt_auth_failed",
+          "Deepgram STT: missing API key. Set DEEPGRAM_API_KEY in the agent env.",
+        );
+      }
+
+      const client = new DeepgramClient({ apiKey });
+      let connection: V1Socket;
+      try {
+        connection = await client.listen.v1.connect({
+          model: opts.model ?? "nova-3",
+          language: opts.language ?? "en",
+          encoding: "linear16",
+          sample_rate: openOpts.sampleRate,
+          channels: 1,
+          interim_results: "true",
+          smart_format: "true",
+          endpointing: 300,
+          utterance_end_ms: "1000",
+          // Pass the API key explicitly as the Authorization header so the
+          // WebSocket connection authenticates even without env var fallback.
+          Authorization: apiKey,
+        });
+      } catch (cause) {
+        throw makeSttError(
+          "stt_connect_failed",
+          `Deepgram STT: connect failed: ${cause instanceof Error ? cause.message : String(cause)}`,
+        );
+      }
+
+      const emitter: Emitter<SttEvents> = createNanoEvents<SttEvents>();
+      let closed = false;
+
+      wireSocketEvents(connection, emitter, () => closed);
+
+      // Actually open the WebSocket connection (registers internal handlers
+      // and initiates the TCP/TLS handshake).
+      connection.connect();
+      try {
+        await connection.waitForOpen();
+      } catch (cause) {
+        throw makeSttError(
+          "stt_connect_failed",
+          `Deepgram STT: WebSocket open failed: ${cause instanceof Error ? cause.message : String(cause)}`,
+        );
+      }
+
+      const close = async (): Promise<void> => {
+        if (closed) return;
+        closed = true;
+        try {
+          connection.close();
+        } catch {
+          // Swallow: the caller has already decided to tear down.
+        }
+      };
+
+      wireAbortSignal(openOpts.signal, close);
+
+      const session: DeepgramSession = {
+        sendAudio(pcm: Int16Array) {
+          if (closed) return;
+          connection.sendMedia(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength));
+        },
+        on(event, fn) {
+          return emitter.on(event, fn);
+        },
+        close,
+        _connection: connection,
+      };
+
+      return session;
+    },
+  };
+}

package/host/providers/tts/cartesia.ts

@@ -24,7 +24,10 @@ import { randomUUID } from "node:crypto";
 import { Cartesia } from "@cartesia/cartesia-js";
 import type { TTSWS, TTSWSContext } from "@cartesia/cartesia-js/resources/tts";
 import { createNanoEvents, type Emitter } from "nanoevents";
-import type { CartesiaOptions } from "../../../sdk/providers/tts/cartesia.ts";
+import {
+  CARTESIA_DEFAULT_VOICE,
+  type CartesiaOptions,
+} from "../../../sdk/providers/tts/cartesia.ts";
 import {
   makeTtsError,
   type TtsEvents,
@@ -73,6 +76,7 @@ export function openCartesia(opts: CartesiaOptions): TtsOpener {
       const sampleRate = assertSupportedSampleRate(openOpts.sampleRate);
       const model = opts.model ?? "sonic-2";
       const language = opts.language ?? "en";
+      const voice = opts.voice ?? CARTESIA_DEFAULT_VOICE;
 
       const client = new Cartesia({ apiKey });
       let ws: TTSWS;
@@ -92,7 +96,7 @@ export function openCartesia(opts: CartesiaOptions): TtsOpener {
       const mintContext = (): TTSWSContext =>
         ws.context({
           model_id: model,
-          voice: { mode: "id", id: opts.voice },
+          voice: { mode: "id", id: voice },
           output_format: {
             container: "raw",
             encoding: "pcm_s16le",
@@ -175,7 +179,7 @@ export function openCartesia(opts: CartesiaOptions): TtsOpener {
 
       const baseRequest = {
         model_id: model,
-        voice: { mode: "id" as const, id: opts.voice },
+        voice: { mode: "id" as const, id: voice },
         output_format: {
           container: "raw" as const,
           encoding: "pcm_s16le" as const,

package/host/providers/tts/rime.test.ts

@@ -0,0 +1,251 @@
+// Copyright 2026 the AAI authors. MIT license.
+/** Unit test for the Rime TTS adapter. Mocks the `ws` package. */
+
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { openRime, type RimeSession } from "./rime.ts";
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Fake WebSocket — hoisted so `vi.mock` factory can reference it
+// ──────────────────────────────────────────────────────────────────────────────
+
+type WsEvent = "open" | "message" | "error" | "close";
+type WsListener = (...args: unknown[]) => void;
+
+const { FakeWebSocket } = vi.hoisted(() => {
+  class FakeWebSocket {
+    static OPEN = 1;
+    static CLOSED = 3;
+
+    readyState = FakeWebSocket.OPEN;
+    sent: string[] = [];
+    private readonly listeners = new Map<string, WsListener[]>();
+
+    static instances: FakeWebSocket[] = [];
+
+    readonly url: string;
+
+    constructor(url: string, _opts?: unknown) {
+      this.url = url;
+      FakeWebSocket.instances.push(this);
+      // Simulate async open on next microtask (matches real ws behaviour).
+      queueMicrotask(() => this._fire("open"));
+    }
+
+    on(event: string, fn: WsListener) {
+      const arr = this.listeners.get(event) ?? [];
+      arr.push(fn);
+      this.listeners.set(event, arr);
+    }
+
+    once(event: string, fn: WsListener) {
+      const wrapper = (...args: unknown[]) => {
+        this.off(event, wrapper);
+        fn(...args);
+      };
+      this.on(event, wrapper);
+    }
+
+    removeListener(event: string, fn: WsListener) {
+      this.off(event, fn);
+    }
+
+    private off(event: string, fn: WsListener) {
+      const arr = this.listeners.get(event) ?? [];
+      this.listeners.set(
+        event,
+        arr.filter((l) => l !== fn),
+      );
+    }
+
+    send(data: string) {
+      this.sent.push(data);
+    }
+
+    close() {
+      this.readyState = FakeWebSocket.CLOSED;
+      this._fire("close");
+    }
+
+    /** Test helper: fire an event on this socket. */
+    _fire(event: WsEvent, ...args: unknown[]) {
+      for (const fn of this.listeners.get(event) ?? []) fn(...args);
+    }
+
+    /** Test helper: simulate a JSON message from the server. */
+    _msg(payload: unknown) {
+      this._fire("message", JSON.stringify(payload));
+    }
+  }
+
+  return { FakeWebSocket };
+});
+
+vi.mock("ws", () => ({
+  default: FakeWebSocket,
+  WebSocket: FakeWebSocket,
+}));
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ──────────────────────────────────────────────────────────────────────────────
+
+beforeEach(() => {
+  FakeWebSocket.instances.length = 0;
+  vi.useFakeTimers();
+});
+
+afterEach(() => {
+  vi.useRealTimers();
+});
+
+async function openSession(apiKey = "test-key"): Promise<{
+  session: RimeSession;
+  ws: InstanceType<typeof FakeWebSocket>;
+  controller: AbortController;
+}> {
+  const opener = openRime({ voice: "cove" });
+  const controller = new AbortController();
+
+  const openPromise = opener.open({
+    sampleRate: 16_000,
+    apiKey,
+    signal: controller.signal,
+  }) as Promise<RimeSession>;
+
+  // Let the microtask that fires FakeWebSocket "open" run.
+  await Promise.resolve();
+
+  const session = await openPromise;
+  // biome-ignore lint/style/noNonNullAssertion: at(-1) is always set after open() resolves
+  const ws = FakeWebSocket.instances.at(-1)!;
+  return { session, ws, controller };
+}
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Tests
+// ──────────────────────────────────────────────────────────────────────────────
+
+describe("rime TTS adapter", () => {
+  test("openRime returns an opener with name 'rime'", () => {
+    const opener = openRime({ voice: "cove" });
+    expect(opener.name).toBe("rime");
+  });
+
+  test("open() throws tts_auth_failed when API key is missing", async () => {
+    const opener = openRime({ voice: "cove" });
+    const controller = new AbortController();
+
+    const openPromise = opener.open({
+      sampleRate: 16_000,
+      apiKey: "",
+      signal: controller.signal,
+    });
+
+    await expect(openPromise).rejects.toMatchObject({ code: "tts_auth_failed" });
+  });
+
+  test("incoming chunk message emits audio as Int16Array", async () => {
+    const { session, ws } = await openSession();
+
+    const audioEvents: Int16Array[] = [];
+    session.on("audio", (pcm) => audioEvents.push(pcm));
+
+    // Encode 4 PCM16 samples (8 bytes) as base64.
+    const samples = new Int16Array([100, 200, 300, 400]);
+    const base64 = Buffer.from(samples.buffer).toString("base64");
+
+    ws._msg({ type: "chunk", data: base64, contextId: null });
+
+    expect(audioEvents.length).toBe(1);
+    const firstChunk = audioEvents[0];
+    expect(firstChunk).toBeInstanceOf(Int16Array);
+    // Each sample pair decodes correctly.
+    // biome-ignore lint/style/noNonNullAssertion: length was asserted to be 1 on the line above
+    const pcm = firstChunk!;
+    expect(pcm.length).toBe(4);
+    expect(pcm[0]).toBe(100);
+    expect(pcm[3]).toBe(400);
+  });
+
+  test("sendText forwards the text as a JSON {text} frame", async () => {
+    const { session, ws } = await openSession();
+
+    session.sendText("Hello, world!");
+
+    expect(ws.sent).toContain(JSON.stringify({ text: "Hello, world!" }));
+  });
+
+  test("flush() sends a trailing '.' and emits done after quiescence post-audio", async () => {
+    const { session, ws } = await openSession();
+
+    const doneEvents: number[] = [];
+    session.on("done", () => doneEvents.push(Date.now()));
+
+    session.sendText("Hi there");
+    session.flush();
+
+    // Trailing punctuation forces Rime to synthesize the buffer without
+    // closing the WS (which `eos` would do).
+    expect(ws.sent).toContain(JSON.stringify({ text: "." }));
+
+    // First-audio timer is 5 s — short window must not fire `done` yet.
+    vi.advanceTimersByTime(500);
+    expect(doneEvents.length).toBe(0);
+
+    // First chunk arrives → switch to short quiescence window.
+    const samples = new Int16Array([100, 200, 300, 400]);
+    ws._msg({
+      type: "chunk",
+      data: Buffer.from(samples.buffer).toString("base64"),
+      contextId: null,
+    });
+
+    vi.advanceTimersByTime(499);
+    expect(doneEvents.length).toBe(0);
+    vi.advanceTimersByTime(1);
+    expect(doneEvents.length).toBe(1);
+  });
+
+  test("flush() falls back to first-audio timeout when no chunk arrives", async () => {
+    const { session } = await openSession();
+
+    const doneEvents: number[] = [];
+    session.on("done", () => doneEvents.push(Date.now()));
+
+    session.sendText("Hi there");
+    session.flush();
+
+    // No chunk arrives — done must wait the full FIRST_AUDIO_TIMEOUT_MS (5 s).
+    vi.advanceTimersByTime(4999);
+    expect(doneEvents.length).toBe(0);
+    vi.advanceTimersByTime(1);
+    expect(doneEvents.length).toBe(1);
+  });
+
+  test("cancel() sends clear operation and emits done synchronously", async () => {
+    const { session, ws } = await openSession();
+
+    const doneEvents: number[] = [];
+    session.on("done", () => doneEvents.push(Date.now()));
+
+    session.sendText("Hello");
+    // cancel() must emit `done` synchronously — barge-in cannot be deferred.
+    session.cancel();
+
+    expect(ws.sent).toContain(JSON.stringify({ operation: "clear" }));
+    // done was emitted synchronously (before any await / timer).
+    expect(doneEvents.length).toBe(1);
+  });
+
+  test("close() closes the WebSocket and is idempotent", async () => {
+    const { session, ws } = await openSession();
+
+    expect(ws.readyState).toBe(FakeWebSocket.OPEN);
+
+    await session.close();
+    expect(ws.readyState).toBe(FakeWebSocket.CLOSED);
+
+    // Second close should not throw.
+    await expect(session.close()).resolves.toBeUndefined();
+  });
+});